send.py should write everything to stdout (without buffering)

common.py

@@ -73,14 +73,17 @@ def dumps(sym, n=1):
     return data * n
 
 
-def iterate(data, size, func=None):
+def iterate(data, size, func=None, truncate=True):
     offset = 0
     data = iter(data)
 
-    while True:
+    done = False
+    while not done:
         buf = list(itertools.islice(data, size))
         if len(buf) < size:
-            return
+            if truncate or not buf:
+                return
+            done = True
 
         buf = np.array(buf)
         result = func(buf) if func else buf

ecc.py

@@ -1,6 +1,8 @@
 ''' Reed-Solomon CODEC. '''
 from reedsolo import rs_encode_msg, rs_correct_msg
 
+import common
+
 import logging
 log = logging.getLogger(__name__)
 
@@ -15,12 +17,10 @@ def end_of_stream(size):
 def encode(data, nsym=DEFAULT_NSYM):
     chunk_size = BLOCK_SIZE - nsym - 1
 
-    for i in range(0, len(data), chunk_size):
-        chunk = bytearray(data[i:i+chunk_size])
-
+    for _, chunk in common.iterate(data, chunk_size, bytearray, truncate=False):
         size = len(chunk)
         if size < chunk_size:
-            padding = b'\x00' * (chunk_size - size)
+            padding = [0] * (chunk_size - size)
             chunk.extend(padding)
 
         block = bytearray([size]) + chunk

send.py

@@ -9,6 +9,8 @@ log = logging.getLogger(__name__)
 import sigproc
 import train
 import wave
+import stream
+
 from common import *
 
 
@@ -17,36 +19,44 @@ class Symbol(object):
     carrier = [np.exp(2j * np.pi * F * t) for F in frequencies]
 
 sym = Symbol()
-data = sys.stdin.read()
 
 
 def write(fd, sym, n=1):
     fd.write(dumps(sym, n))
 
 
-def start(sig, c):
-    write(sig, c*0, n=50)
-    write(sig, c*1, n=400)
-    write(sig, c*0, n=50)
+def start(fd, c):
+    write(fd, c*0, n=50)
+    write(fd, c*1, n=400)
+    write(fd, c*0, n=50)
 
 
-def training(sig, c):
+def training(fd, c):
     for b in train.equalizer:
-        write(sig, c * b)
+        write(fd, c * b)
 
 
-def modulate(sig, bits):
+def modulate(fd, bits):
     symbols_iter = sigproc.modulator.encode(bits)
     symbols_iter = itertools.chain(symbols_iter, itertools.repeat(0))
     carriers = np.array(sym.carrier) / len(sym.carrier)
     while True:
         symbols = itertools.islice(symbols_iter, len(sym.carrier))
         symbols = np.array(list(symbols))
-        write(sig, np.dot(symbols, carriers))
+        write(fd, np.dot(symbols, carriers))
         if all(symbols == 0):
             break
 
 
+def iread(fd, size):
+    while True:
+        block = fd.read(size)
+        if block:
+            yield block
+        else:
+            return
+
+
 def main(fname):
     import ecc
     log.info('Running MODEM @ {:.1f} kbps'.format(sigproc.modem_bps / 1e3))
@@ -59,14 +69,10 @@ def main(fname):
         log.info('%.3f seconds of training audio',
                  training_size / wave.bytes_per_second)
 
-        log.info('%d bits to be send', len(data)*8)
-        buf = bytearray()
-        for block in ecc.encode(data):
-            buf.extend(block)
-        bits = list(to_bits(buf))
-        ratio = 1.0 - len(data)*8.0 / len(bits)
-        log.info('%d bits modulated (%.1f%% overhead)', len(bits), ratio * 100)
-        modulate(fd, bits)
+        data = itertools.chain.from_iterable(iread(sys.stdin, 64 << 10))
+        encoded = itertools.chain.from_iterable(ecc.encode(data))
+        modulate(fd, bits=to_bits(encoded))
+
         data_size = fd.tell() - training_size
         log.info('%.3f seconds of data audio',
                  data_size / wave.bytes_per_second)

sigproc.py

@@ -45,13 +45,8 @@ class QAM(object):
         self.bits_per_symbol = bits_per_symbol
 
     def encode(self, bits):
-        trailing_bits = len(bits) % self.bits_per_symbol
-        if trailing_bits:
-            bits = bits + [0] * (self.bits_per_symbol - trailing_bits)
-        for i in range(0, len(bits), self.bits_per_symbol):
-            bits_tuple = tuple(bits[i:i+self.bits_per_symbol])
-            s = self._enc[bits_tuple]
-            yield s
+        for _, bits_tuple in common.iterate(bits, self.bits_per_symbol, tuple):
+            yield self._enc[bits_tuple]
 
     def decode(self, symbols, error_handler=None):
         for s in symbols: