Add python bindings + package (#10)

* wip : python package

* wip : minor fixes

* wip : upload package to main pypi

* wip : initial text encoding

* wip : extending C api

* wip : use map of global instances

* wip : added decode functionality

* update main README
This commit is contained in:
Georgi Gerganov
2021-01-17 17:36:50 +02:00
committed by GitHub
parent 94978e679a
commit 2ed431fa81
16 changed files with 701 additions and 9 deletions

View File

@@ -92,8 +92,9 @@ The [examples](https://github.com/ggerganov/ggwave/blob/master/examples/) folder
| [ggwave-rx](https://github.com/ggerganov/ggwave/blob/master/examples/ggwave-rx) | Very basic receive-only program | SDL |
| [ggwave-cli](https://github.com/ggerganov/ggwave/blob/master/examples/ggwave-cli) | Command line tool for sending/receiving data through sound | SDL |
| [ggwave-wasm](https://github.com/ggerganov/ggwave/blob/master/examples/ggwave-wasm) | WebAssembly module for web applications | SDL |
| [ggwave-to-file](https://github.com/ggerganov/ggwave/blob/master/examples/ggwave-to-file) | Output a generated waveform to an uncompressed WAV file | - |
| [ggwave-to-file](https://github.com/ggerganov/ggwave/blob/master/examples/ggwave-to-file) | Output a generated waveform to an uncompressed WAV file | - |
| [waver](https://github.com/ggerganov/ggwave/blob/master/examples/waver) | GUI application for sending/receiving data through sound | SDL |
| [ggwave-py](https://github.com/ggerganov/ggwave/blob/master/examples/ggwave-py) | Python examples | - |
Other projects using **ggwave** or one of its prototypes:
@@ -137,6 +138,13 @@ emcmake cmake ..
make
```
### Python
```bash
pip install ggwave
```
More info: https://pypi.org/project/ggwave/
## Installing the Waver application

1
bindings/python/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
ggwave.so

View File

@@ -0,0 +1,4 @@
recursive-include ggwave/include *
recursive-include ggwave/src/ *
include *.pxd
include *.pyx

41
bindings/python/Makefile Normal file
View File

@@ -0,0 +1,41 @@
default: build
.PHONY:
ggwave:
# create a clean (maybe updated) copy of ggwave src
rm -rf ggwave && mkdir ggwave && cp -r ../../include ggwave/ && cp -r ../../src ggwave/
pyggwave.bycython.cpp: ggwave.pyx cggwave.pxd
python -m pip install cython
cython --cplus ggwave.pyx -o ggwave.bycython.cpp
# To build package, README.rst is needed, because it goes into long description of package,
# which is what is visible on PyPI.
# However, to generate README.rst from README-tmpl.rst, built package is needed (for `import ggwave` in cog)!
# Therefore, we first build package without README.rst, use it to generate README.rst,
# and then finally build package again but with README.rst.
BUILD_SOURCE_FILES=ggwave pyggwave.bycython.cpp setup.py
buildWithoutREADME.rst: ${BUILD_SOURCE_FILES}
GGWAVE_OMIT_README_RST=1 python setup.py build_ext -i
README.rst: buildWithoutREADME.rst README-tmpl.rst
python -m pip install cogapp
cog -d -o README.rst README-tmpl.rst
BUILD_FILES=${BUILD_SOURCE_FILES} README.rst
build: ${BUILD_FILES}
python setup.py build_ext -i
sdist: ggwave pyggwave.bycython.cpp setup.py README.rst MANIFEST.in
python setup.py sdist
publish: clean sdist
twine upload --repository pypi dist/*
clean:
rm -rf ggwave dist ggwave.egg-info build
rm -f ggwave.c *.bycython.* ggwave.*.so
rm -f README.rst

View File

@@ -0,0 +1,195 @@
.. [[[cog
import cog
import ggwave
def indent(text, indentation = " "):
return indentation + text.replace("\n", "\n" + indentation)
def comment(text):
return "# " + text.replace("\n", "\n# ")
def cogOutExpression(expr):
cog.outl(indent(expr))
cog.outl(indent(comment(str(eval(expr)))))
]]]
[[[end]]]
======
ggwave
======
Tiny data-over-sound library.
.. [[[cog
cog.outl()
cog.outl(".. code:: python")
cog.outl()
cog.outl(indent(comment('generate audio waveform for string "hello python"')))
cog.outl(indent('waveform = ggwave.encode("hello python")'))
cog.outl()
cog.outl(indent(comment('decode audio waveform')))
cog.outl(indent('text = ggwave.decode(instance, waveform)'))
cog.outl()
]]]
.. code::
{{ Basic code examples will be generated here. }}
.. [[[end]]]
--------
Features
--------
* Audible and ultrasound transmissions available
* Bandwidth of 8-16 bytes/s (depending on the transmission protocol)
* Robust FSK modulation
* Reed-Solomon based error correction
------------
Installation
------------
::
pip install ggwave
---
API
---
encode()
--------
.. code:: python
encode(payload, [txProtocol], [volume], [instance])
Encodes ``payload`` into an audio waveform.
.. [[[cog
import pydoc
help_str = pydoc.plain(pydoc.render_doc(ggwave.encode, "%s"))
cog.outl()
cog.outl('Output of ``help(ggwave.encode)``:')
cog.outl()
cog.outl('.. code::\n')
cog.outl(indent(help_str))
]]]
.. code::
{{ Content of help(ggwave.encode) will be generated here. }}
.. [[[end]]]
decode()
--------
.. code:: python
decode(instance, waveform)
Analyzes and decodes ``waveform`` into to try and obtain the original payload.
A preallocated ggwave ``instance`` is required.
.. [[[cog
import pydoc
help_str = pydoc.plain(pydoc.render_doc(ggwave.decode, "%s"))
cog.outl()
cog.outl('Output of ``help(ggwave.decode)``:')
cog.outl()
cog.outl('.. code::\n')
cog.outl(indent(help_str))
]]]
.. code::
{{ Content of help(ggwave.decode) will be generated here. }}
.. [[[end]]]
-----
Usage
-----
* Encode and transmit data with sound:
.. code:: python
import ggwave
import pyaudio
import numpy as np
p = pyaudio.PyAudio()
# generate audio waveform for string "hello python"
waveform = ggwave.encode("hello python", txProtocol = 1, volume = 20)
print("Transmitting text 'hello python' ...")
stream = p.open(format=pyaudio.paInt16, channels=1, rate=48000, output=True, frames_per_buffer=4096)
stream.write(np.array(waveform).astype(np.int16), len(waveform))
stream.stop_stream()
stream.close()
p.terminate()
* Capture and decode audio data:
.. code:: python
import ggwave
import pyaudio
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paFloat32, channels=1, rate=48000, input=True, frames_per_buffer=1024)
print('Listening ... Press Ctrl+C to stop')
instance = ggwave.init()
try:
while True:
data = stream.read(1024)
res = ggwave.decode(instance, data)
if (not res is None):
try:
print('Received text: ' + res.decode("utf-8"))
except:
pass
except KeyboardInterrupt:
pass
ggwave.free(instance)
stream.stop_stream()
stream.close()
p.terminate()
----
More
----
Check out `<http://github.com/ggerganov/ggwave>`_ for more information about ggwave!
-----------
Development
-----------
Check out `ggwave python package on Github <https://github.com/ggerganov/ggwave/tree/master/bindings/python>`_.

34
bindings/python/README.md Normal file
View File

@@ -0,0 +1,34 @@
# ggwave python package
This README contains only development information, you can check out full README (README.rst) for the latest version of ggwave python package on [ggwave's PyPI page](https://pypi.org/project/ggwave/).
README.rst is not commited to git because it is generated from [README-tmpl.rst](./README-tmpl.rst).
## Building
Run `make build` to generate an extension module as .so file.
You can test it then by importing it from python interpreter `import ggwave` and running `ggwave.testC(...)` (you have to be positioned in the directory where .so was built).
This is useful for testing while developing.
Run `make sdist` to create a source distribution, but not publish it - it is a tarball in dist/ that will be uploaded to pip on `publish`.
Use this to check that tarball is well structured and contains all needed files, before you publish.
Good way to test it is to run `sudo pip install dist/ggwave-*.tar.gz`, which will try to install ggwave from it, same way as pip will do it when it is published.
`make clean` removes all generated files.
README.rst is auto-generated from [README-tmpl.rst](./README-tmpl.rst), to run regeneration do `make README.rst`.
README.rst is also automatically regenerated when building package (e.g. `make build`).
This enables us to always have up to date results of code execution and help documentation of ggwave methods in readme.
## Publishing
Remember to update version in setup.py before publishing.
To trigger automatic publish to PyPI, create a tag and push it to Github -> Travis will create sdist, build wheels, and push them all to PyPI while publishing new version.
You can also publish new version manually if needed: run `make publish` to create a source distribution and publish it to the PyPI.
## Acknowledgments
These Python bindings are generated by following [edlib](https://github.com/Martinsos/edlib) example

View File

@@ -0,0 +1,42 @@
cdef extern from "ggwave.h" nogil:
ctypedef enum ggwave_SampleFormat:
GGWAVE_SAMPLE_FORMAT_I16,
GGWAVE_SAMPLE_FORMAT_F32
ctypedef enum ggwave_TxProtocol:
GGWAVE_TX_PROTOCOL_AUDIBLE_NORMAL,
GGWAVE_TX_PROTOCOL_AUDIBLE_FAST,
GGWAVE_TX_PROTOCOL_AUDIBLE_FASTEST,
GGWAVE_TX_PROTOCOL_ULTRASOUND_NORMAL,
GGWAVE_TX_PROTOCOL_ULTRASOUND_FAST,
GGWAVE_TX_PROTOCOL_ULTRASOUND_FASTEST
ctypedef struct ggwave_Parameters:
int sampleRateIn
int sampleRateOut
int samplesPerFrame
ggwave_SampleFormat formatIn
ggwave_SampleFormat formatOut
ctypedef int ggwave_Instance
ggwave_Parameters ggwave_defaultParameters();
ggwave_Instance ggwave_init(const ggwave_Parameters parameters);
void ggwave_free(ggwave_Instance instance);
int ggwave_encode(
ggwave_Instance instance,
const char * dataBuffer,
int dataSize,
ggwave_TxProtocol txProtocol,
int volume,
char * outputBuffer);
int ggwave_decode(
ggwave_Instance instance,
const char * dataBuffer,
int dataSize,
char * outputBuffer);

View File

@@ -0,0 +1,65 @@
cimport cython
from cpython.mem cimport PyMem_Malloc, PyMem_Free
import re
import struct
cimport cggwave
def defaultParameters():
return cggwave.ggwave_defaultParameters()
def init(parameters = None):
if (parameters is None):
parameters = defaultParameters()
return cggwave.ggwave_init(parameters)
def free(instance):
return cggwave.ggwave_free(instance)
def encode(payload, txProtocol = 1, volume = 10, instance = None):
""" Encode payload into an audio waveform.
@param {string} payload, the data to be encoded
@return Generated audio waveform bytes representing 16-bit signed integer samples.
"""
cdef bytes data_bytes = payload.encode()
cdef char* cdata = data_bytes
cdef bytes output_bytes = bytes(1024*1024)
cdef char* coutput = output_bytes
own = False
if (instance is None):
own = True
instance = init(defaultParameters())
n = cggwave.ggwave_encode(instance, cdata, len(data_bytes), txProtocol, volume, coutput)
if (own):
free(instance)
# add short silence at the end
n += 16*1024
return struct.unpack("h"*n, output_bytes[0:2*n])
def decode(instance, waveform):
""" Analyze and decode audio waveform to obtain original payload
@param {bytes} waveform, the audio waveform to decode
@return The decoded payload if successful.
"""
cdef bytes data_bytes = waveform
cdef char* cdata = data_bytes
cdef bytes output_bytes = bytes(256)
cdef char* coutput = output_bytes
rxDataLength = cggwave.ggwave_decode(instance, cdata, len(data_bytes), coutput)
if (rxDataLength > 0):
return coutput[0:rxDataLength]
return None

47
bindings/python/setup.py Normal file
View File

@@ -0,0 +1,47 @@
from setuptools import setup, Extension
from codecs import open
import os
cmdclass = {}
long_description = ""
# Build directly from cython source file(s) if user wants so (probably for some experiments).
# Otherwise, pre-generated c source file(s) are used.
# User has to set environment variable GGWAVE_USE_CYTHON.
# e.g.: GGWAVE_USE_CYTHON=1 python setup.py install
USE_CYTHON = os.getenv('GGWAVE_USE_CYTHON', False)
if USE_CYTHON:
from Cython.Build import build_ext
ggwave_module_src = "ggwave.pyx"
cmdclass['build_ext'] = build_ext
else:
ggwave_module_src = "ggwave.bycython.cpp"
# Load README.rst into long description.
# User can skip using README.rst as long description: GGWAVE_OMIT_README_RST=1 python setup.py install
OMIT_README_RST = os.getenv('GGWAVE_OMIT_README_RST', False)
if not OMIT_README_RST:
here = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
setup(
# Information
name = "ggwave",
description = "Tiny data-over-sound library.",
long_description = long_description,
version = "0.1.3",
url = "https://github.com/ggerganov/ggwave",
author = "Georgi Gerganov",
author_email = "ggerganov@gmail.com",
license = "MIT",
keywords = "data-over-sound fsk ecc serverless pairing qrcode ultrasound",
# Build instructions
ext_modules = [Extension("ggwave",
[ggwave_module_src, "ggwave/src/ggwave.cpp"],
include_dirs=["ggwave/include", "ggwave/include/ggwave"],
depends=["ggwave/include/ggwave/ggwave.h"],
language="c++",
extra_compile_args=["-O3", "-std=c++11"])],
cmdclass = cmdclass
)

16
bindings/python/test.py Normal file
View File

@@ -0,0 +1,16 @@
import sys
import ggwave
testFailed = False
n, samples = ggwave.encode("hello python")
if not (samples and n > 1024):
testFailed = True
if testFailed:
print("Some of the tests failed!")
else:
print("All tests passed!")
sys.exit(testFailed)

View File

@@ -0,0 +1,9 @@
## ggwave-py
Python examples using the `ggwave` python package
### Install
```bash
pip install ggwave
```

View File

@@ -0,0 +1,28 @@
import ggwave
import pyaudio
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paFloat32, channels=1, rate=48000, input=True, frames_per_buffer=1024)
print('Listening ... Press Ctrl+C to stop')
instance = ggwave.init()
try:
while True:
data = stream.read(1024)
res = ggwave.decode(instance, data)
if (not res is None):
try:
print('Received text: ' + res.decode("utf-8"))
except:
pass
except KeyboardInterrupt:
pass
ggwave.free(instance)
stream.stop_stream()
stream.close()
p.terminate()

View File

@@ -0,0 +1,16 @@
import ggwave
import pyaudio
import numpy as np
p = pyaudio.PyAudio()
# generate audio waveform for string "hello python"
waveform = ggwave.encode("hello python", txProtocol = 1, volume = 20)
print("Transmitting text 'hello python' ...")
stream = p.open(format=pyaudio.paInt16, channels=1, rate=48000, output=True, frames_per_buffer=4096)
stream.write(np.array(waveform).astype(np.int16), len(waveform))
stream.stop_stream()
stream.close()
p.terminate()

View File

@@ -1,4 +1,71 @@
#pragma once
#ifndef GGWAVE_H
#define GGWAVE_H
// Define GGWAVE_API macro to properly export symbols
#ifdef GGWAVE_SHARED
# ifdef _WIN32
# ifdef GGWAVE_BUILD
# define GGWAVE_API __declspec(dllexport)
# else
# define GGWAVE_API __declspec(dllimport)
# endif
# else
# define GGWAVE_API __attribute__ ((visibility ("default")))
# endif
#else
# define GGWAVE_API
#endif
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
GGWAVE_SAMPLE_FORMAT_I16,
GGWAVE_SAMPLE_FORMAT_F32,
} ggwave_SampleFormat;
typedef enum {
GGWAVE_TX_PROTOCOL_AUDIBLE_NORMAL,
GGWAVE_TX_PROTOCOL_AUDIBLE_FAST,
GGWAVE_TX_PROTOCOL_AUDIBLE_FASTEST,
GGWAVE_TX_PROTOCOL_ULTRASOUND_NORMAL,
GGWAVE_TX_PROTOCOL_ULTRASOUND_FAST,
GGWAVE_TX_PROTOCOL_ULTRASOUND_FASTEST,
} ggwave_TxProtocol;
typedef struct {
int sampleRateIn;
int sampleRateOut;
int samplesPerFrame;
ggwave_SampleFormat formatIn;
ggwave_SampleFormat formatOut;
} ggwave_Parameters;
typedef int ggwave_Instance;
GGWAVE_API ggwave_Parameters ggwave_defaultParameters(void);
GGWAVE_API ggwave_Instance ggwave_init(const ggwave_Parameters parameters);
GGWAVE_API void ggwave_free(ggwave_Instance instance);
GGWAVE_API int ggwave_encode(
ggwave_Instance instance,
const char * dataBuffer,
int dataSize,
ggwave_TxProtocol txProtocol,
int volume,
char * outputBuffer);
GGWAVE_API int ggwave_decode(
ggwave_Instance instance,
const char * dataBuffer,
int dataSize,
char * outputBuffer);
#ifdef __cplusplus
}
#include <cstdint>
#include <functional>
@@ -6,7 +73,7 @@
class GGWave {
public:
static constexpr auto kBaseSampleRate = 48000.0;
static constexpr auto kBaseSampleRate = 48000;
static constexpr auto kDefaultSamplesPerFrame = 1024;
static constexpr auto kMaxSamplesPerFrame = 1024;
static constexpr auto kMaxDataBits = 256;
@@ -46,6 +113,7 @@ public:
using RecordedData = std::vector<float>;
using TxRxData = std::vector<std::uint8_t>;
// todo : rename to CBEnqueueAudio
using CBQueueAudio = std::function<void(const void * data, uint32_t nBytes)>;
using CBDequeueAudio = std::function<uint32_t(void * data, uint32_t nMaxBytes)>;
@@ -59,6 +127,8 @@ public:
~GGWave();
bool init(int textLength, const char * stext, const TxProtocol & aProtocol, const int volume);
// todo : rename to "encode" / "decode"
bool send(const CBQueueAudio & cbQueueAudio);
void receive(const CBDequeueAudio & CBDequeueAudio);
@@ -77,7 +147,8 @@ public:
const float & getSampleRateIn() const { return m_sampleRateIn; }
const float & getSampleRateOut() const { return m_sampleRateOut; }
const TxProtocol & getDefultTxProtocol() const { return getTxProtocols()[1]; }
static int getDefultTxProtocolId() { return 1; }
static const TxProtocol & getDefultTxProtocol() { return getTxProtocols()[getDefultTxProtocolId()]; }
const TxRxData & getRxData() const { return m_rxData; }
const TxProtocol & getRxProtocol() const { return m_rxProtocol; }
@@ -160,3 +231,7 @@ private:
AmplitudeData16 m_outputBlock16;
AmplitudeData16 m_txAmplitudeData16;
};
#endif
#endif

View File

@@ -1,16 +1,22 @@
# core
add_library(ggwave ${GGWAVE_LIBRARY_TYPE}
set(TARGET ggwave)
add_library(${TARGET} ${GGWAVE_LIBRARY_TYPE}
ggwave.cpp
)
target_include_directories(ggwave PUBLIC
target_include_directories(${TARGET} PUBLIC
.
../include
)
if (GGWAVE_LIBRARY_TYPE STREQUAL "SHARED")
target_link_libraries(ggwave PUBLIC
target_link_libraries(${TARGET} PUBLIC
${CMAKE_DL_LIBS}
)
target_compile_definitions(${TARGET} PUBLIC
GGWAVE_SHARED
)
endif()

View File

@@ -6,6 +6,111 @@
#include <algorithm>
#include <random>
#include <stdexcept>
#include <map>
//
// C interface
//
namespace {
std::map<ggwave_Instance, GGWave *> g_instances;
}
extern "C"
ggwave_Parameters ggwave_defaultParameters(void) {
ggwave_Parameters result {
GGWave::kBaseSampleRate,
GGWave::kBaseSampleRate,
GGWave::kDefaultSamplesPerFrame,
GGWAVE_SAMPLE_FORMAT_F32,
GGWAVE_SAMPLE_FORMAT_I16
};
return result;
}
extern "C"
ggwave_Instance ggwave_init(const ggwave_Parameters parameters) {
static ggwave_Instance curId = 0;
g_instances[curId] = new GGWave(
parameters.sampleRateIn,
parameters.sampleRateOut,
parameters.samplesPerFrame,
4, // todo : hardcoded sample sizes
2);
return curId++;
}
extern "C"
void ggwave_free(ggwave_Instance instance) {
delete (GGWave *) g_instances[instance];
g_instances.erase(instance);
}
extern "C"
int ggwave_encode(
ggwave_Instance instance,
const char * dataBuffer,
int dataSize,
ggwave_TxProtocol txProtocol,
int volume,
char * outputBuffer) {
GGWave * ggWave = (GGWave *) g_instances[instance];
ggWave->init(dataSize, dataBuffer, ggWave->getTxProtocols()[txProtocol], volume);
int nSamples = 0;
GGWave::CBQueueAudio cbQueueAudio = [&](const void * data, uint32_t nBytes) {
char * p = (char *) data;
std::copy(p, p + nBytes, outputBuffer);
// todo : tmp assume int16
nSamples = nBytes/2;
};
ggWave->send(cbQueueAudio);
return nSamples;
}
extern "C"
int ggwave_decode(
ggwave_Instance instance,
const char * dataBuffer,
int dataSize,
char * outputBuffer) {
GGWave * ggWave = (GGWave *) g_instances[instance];
GGWave::CBDequeueAudio cbDequeueAudio = [&](void * data, uint32_t nMaxBytes) -> uint32_t {
uint32_t nCopied = std::min((uint32_t) dataSize, nMaxBytes);
std::copy(dataBuffer, dataBuffer + nCopied, (char *) data);
dataSize -= nCopied;
return nCopied;
};
ggWave->receive(cbDequeueAudio);
// todo : avoid allocation
GGWave::TxRxData rxData;
auto rxDataLength = ggWave->takeRxData(rxData);
if (rxDataLength == -1) {
// failed to decode message
return -1;
} else if (rxDataLength > 0) {
std::copy(rxData.begin(), rxData.end(), outputBuffer);
}
return rxDataLength;
}
//
// C++ implementation
//
namespace {
@@ -395,12 +500,12 @@ bool GGWave::send(const CBQueueAudio & cbQueueAudio) {
return true;
}
void GGWave::receive(const CBDequeueAudio & CBDequeueAudio) {
void GGWave::receive(const CBDequeueAudio & cbDequeueAudio) {
while (m_hasNewTxData == false) {
// read capture data
//
// todo : support for non-float input
auto nBytesRecorded = CBDequeueAudio(m_sampleAmplitude.data(), m_samplesPerFrame*m_sampleSizeBytesIn);
auto nBytesRecorded = cbDequeueAudio(m_sampleAmplitude.data(), m_samplesPerFrame*m_sampleSizeBytesIn);
if (nBytesRecorded != 0) {
m_sampleAmplitudeHistory[m_historyId] = m_sampleAmplitude;