luccapirovano/naowalk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

got compression working really well, my laptops wifi card is able to talk to the nao fairly well through the door from the hallway

Browse Source
This commit is contained in:
Lucca Pirovano
2026-07-21 17:59:43 -04:00
parent 7d0b968b36
commit ae9d1652a4
3 changed files with 297 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nao_video_server.py
+94 -66
View File
@@ -3,8 +3,13 @@
nao_video_server.py - RUNS ON THE ROBOT, under its own Python 2.7.
Opens a qi session to NAOqi over localhost (zero network hop - it's
the same box) and serves ONE persistent, length-framed TCP stream
carrying both compressed video (JPEG) and compressed audio (mu-law).
the same box) and serves TWO persistent, length-framed TCP streams -
one for compressed video (JPEG), one for compressed audio (mu-law) -
each on its own port/connection. They used to share one connection,
but a single big video frame mid-send could block audio behind it for
its whole transfer time, which is a bad tradeoff for something as
latency-sensitive as audio. Splitting them means a slow video frame
can never delay audio.
Why not HTTP: the phone tunnel only forwards specific TCP ports, so
this still rides over TCP - but a 5-byte binary header has none of
@@ -18,18 +23,20 @@ whole time. Subscribing to ALAudioDevice locally (same zero-hop trick
as the camera) means only mu-law-compressed bytes ever cross the
tunnel, half the size of raw PCM.
Wire format (all integers big-endian, via `struct`):
Wire format, identical on both connections (all integers big-endian,
via `struct`):
1 byte type 0x00 HELLO 0x01 VIDEO 0x02 AUDIO
4 bytes length
N bytes payload
HELLO payload: JSON {"rate": 16000, "channels": 1}
VIDEO payload: JPEG bytes
AUDIO payload: mu-law encoded 8-bit samples (mono)
HELLO payload: JSON, informational only
VIDEO payload: JPEG bytes (video connection only)
AUDIO payload: mu-law encoded 8-bit samples, mono (audio connection only)
Under load (slow/lossy tunnel), video quality and frame rate step
down automatically (see QUALITY_TIERS) and recover once sends are
fast again. Audio is prioritized over video each pass, since gaps in
audio are far more noticeable than a slightly stale picture.
fast again - and since encoding is throttled to match, this also cuts
CPU use on a congested link instead of just wasting it on frames that
would've been discarded anyway.
Needs the SDK's own env vars, since the bindings aren't on a login
shell's default path:
@@ -54,7 +61,9 @@ import qi
# ---- tunables ----
NAOQI_PORT = 9561 # local qi session port (confirmed working)
TCP_PORT = 8000 # forward this one port through the phone tunnel
VIDEO_PORT = 8000 # forward both of these through the phone tunnel
AUDIO_PORT = 8001 # kept on its own connection so a slow video
# frame can never block/delay audio delivery
CAMERA_RESOLUTION = 1 # 0=kQQVGA(160x120) 1=kQVGA(320x240) 2=kVGA(640x480)
CAMERA_COLORSPACE = 11 # kRGBColorSpace
CAMERA_FPS = 20
@@ -209,66 +218,82 @@ class MicGrabber(object):
pass
class MediaServer(object):
"""Accepts one client at a time and interleaves video + audio onto
its socket. Audio is drained first every pass since gaps in it are
much more noticeable than the picture being a beat stale."""
def _send(conn, msg_type, payload):
conn.sendall(struct.pack(">BI", msg_type, len(payload)))
conn.sendall(payload)
def __init__(self, grabber, mic):
self.grabber = grabber
self.mic = mic
def serve_forever(self, host, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, port))
sock.listen(1)
print("media server listening on :%d" % port)
while True:
conn, addr = sock.accept()
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
conn.settimeout(5.0)
print("client connected: %s" % (addr,))
try:
self._serve_client(conn)
except Exception as e:
print("client disconnected (%s)" % e)
finally:
try:
conn.close()
except Exception:
pass
def _serve_client(self, conn):
self._send(conn, TYPE_HELLO,
json.dumps({"rate": AUDIO_RATE, "channels": AUDIO_CHANNELS}).encode("utf-8"))
def serve_video(grabber, host, port):
"""Video gets its own connection/thread - a slow sendall() here
(a big JPEG frame over a bad link) must never be able to delay
audio, which used to share this same socket and paid for it in
latency."""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, port))
sock.listen(1)
print("video server listening on :%d" % port)
while True:
conn, addr = sock.accept()
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
conn.settimeout(5.0)
print("video client connected: %s" % (addr,))
last_jpeg = None
last_video_send = 0.0
while True:
did_something = False
last_send = 0.0
try:
_send(conn, TYPE_HELLO, json.dumps({"stream": "video"}).encode("utf-8"))
while True:
jpeg = grabber.latest()
_, min_interval = grabber.quality_and_interval()
now = time.time()
if jpeg is not None and jpeg is not last_jpeg and now - last_send >= min_interval:
t0 = time.time()
_send(conn, TYPE_VIDEO, jpeg)
grabber.report_send_time(time.time() - t0)
last_jpeg = jpeg
last_send = now
else:
time.sleep(0.01)
except Exception as e:
print("video client disconnected (%s)" % e)
finally:
try:
conn.close()
except Exception:
pass
if self.mic is not None:
for chunk in self.mic.pop_all():
self._send(conn, TYPE_AUDIO, chunk)
did_something = True
jpeg = self.grabber.latest()
_, min_interval = self.grabber.quality_and_interval()
now = time.time()
if jpeg is not None and jpeg is not last_jpeg and now - last_video_send >= min_interval:
t0 = time.time()
self._send(conn, TYPE_VIDEO, jpeg)
self.grabber.report_send_time(time.time() - t0)
last_jpeg = jpeg
last_video_send = now
did_something = True
if not did_something:
time.sleep(0.01)
def _send(self, conn, msg_type, payload):
conn.sendall(struct.pack(">BI", msg_type, len(payload)))
conn.sendall(payload)
def serve_audio(mic, host, port):
"""Audio's own connection/thread. Checked eagerly (short sleep,
not paced like video) since gaps are far more noticeable than a
video frame being a beat stale."""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, port))
sock.listen(1)
print("audio server listening on :%d" % port)
while True:
conn, addr = sock.accept()
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
conn.settimeout(5.0)
print("audio client connected: %s" % (addr,))
try:
_send(conn, TYPE_HELLO,
json.dumps({"rate": AUDIO_RATE, "channels": AUDIO_CHANNELS}).encode("utf-8"))
while True:
chunks = mic.pop_all() if mic is not None else []
if chunks:
for chunk in chunks:
_send(conn, TYPE_AUDIO, chunk)
else:
time.sleep(0.005)
except Exception as e:
print("audio client disconnected (%s)" % e)
finally:
try:
conn.close()
except Exception:
pass
def main():
@@ -284,9 +309,12 @@ def main():
except Exception as e:
print("mic capture unavailable (%s) - video only" % e)
server = MediaServer(grabber, mic)
video_thread = threading.Thread(target=serve_video, args=(grabber, "0.0.0.0", VIDEO_PORT))
video_thread.daemon = True
video_thread.start()
try:
server.serve_forever("0.0.0.0", TCP_PORT)
serve_audio(mic, "0.0.0.0", AUDIO_PORT)
except KeyboardInterrupt:
pass
finally: