luccapirovano/naowalk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

got compression working really well, my laptops wifi card is able to talk to the nao fairly well through the door from the hallway

Browse Source
This commit is contained in:
Lucca Pirovano
2026-07-21 17:59:43 -04:00
parent 7d0b968b36
commit ae9d1652a4
3 changed files with 297 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
naowalk.py
+201 -22
View File
@@ -347,13 +347,140 @@ class LiveMicRelay:
self._cleanup()
print("🎙️ Talk relay: stopped")
# ==========================================
# ON-ROBOT VIDEO/AUDIO RELAY SERVER (auto-deploy)
# ==========================================
class VideoServerLauncher:
"""
SCPs nao_video_server.py to the robot's home directory over SFTP and
launches it via a persistent, pty-backed SSH session - so the user
never has to manually copy the file over or SSH in to start/stop it.
Uses the same trick as LiveMicRelay: a pty ties the remote process
group to the SSH channel, so close() (killing the channel) reliably
kills the video server too instead of leaving it orphaned on the
robot, still holding VIDEO_PORT/AUDIO_PORT, for next time.
"""
def __init__(self, nao_ip, ssh_user="nao", ssh_password="nao", ssh_port=22,
local_path=None, remote_filename="nao_video_server.py"):
self.nao_ip = nao_ip
self.ssh_user = ssh_user
self.ssh_password = ssh_password
self.ssh_port = ssh_port
self.local_path = local_path or os.path.join(
os.path.dirname(os.path.abspath(__file__)), remote_filename)
self.remote_filename = remote_filename
self.ready = False
self._ssh_client = None
self._channel = None
self._drain_thread = None
def deploy_and_start(self):
if not HAS_SSH:
print("🎥 Video server auto-deploy needs paramiko: pip install paramiko")
return False
if not os.path.isfile(self.local_path):
print(f"🎥 Video server auto-deploy: can't find {self.local_path}")
return False
try:
self._ssh_client = paramiko.SSHClient()
self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self._ssh_client.connect(self.nao_ip, port=self.ssh_port, username=self.ssh_user,
password=self.ssh_password, timeout=10)
# Best-effort: a previous session that didn't get to call
# close() cleanly (laptop lost network, crashed, etc.) can
# leave a stale instance squatting on the ports - clear it
# before starting a fresh one.
try:
_, out, _ = self._ssh_client.exec_command(
f"pkill -f {self.remote_filename}", timeout=5)
out.channel.recv_exit_status()
time.sleep(0.3)
except Exception:
pass
sftp = self._ssh_client.open_sftp()
try:
remote_home = sftp.normalize(".") # robot's home dir
remote_path = remote_home + "/" + self.remote_filename
print(f"🎥 Copying {self.remote_filename} -> {self.nao_ip}:{remote_path}")
sftp.put(self.local_path, remote_path)
finally:
sftp.close()
self._channel = self._ssh_client.get_transport().open_session()
self._channel.get_pty()
remote_cmd = (
"export PYTHONPATH=/opt/aldebaran/lib/python2.7/site-packages && "
"export LD_LIBRARY_PATH=/opt/aldebaran/lib && "
f"python2 {remote_path}"
)
self._channel.exec_command(remote_cmd)
self._drain_thread = threading.Thread(target=self._drain_output, daemon=True)
self._drain_thread.start()
time.sleep(1.0) # give it a moment to bind before the client tries to connect
if self._channel.exit_status_ready():
print("🎥 Video server exited immediately - check the log lines above")
self.ready = False
return False
self.ready = True
print("🎥 Video server deployed and running on robot")
return True
except Exception as e:
print(f"🎥 Video server auto-deploy failed: {e}")
self.ready = False
return False
def _drain_output(self):
"""Keeps reading the remote process's stdout/stderr so its pty
buffer never fills up and stalls the server, and so its own
status prints (quality tier changes, client connects) surface
locally too."""
chan = self._channel
while chan and not chan.closed:
try:
got = False
if chan.recv_ready():
data = chan.recv(4096)
if data:
print(f"🎥 [nao_video_server] {data.decode('utf-8', 'replace').rstrip()}")
got = True
if chan.recv_stderr_ready():
data = chan.recv_stderr(4096)
if data:
print(f"🎥 [nao_video_server] {data.decode('utf-8', 'replace').rstrip()}")
got = True
if not got:
time.sleep(0.1)
except Exception:
break
def close(self):
"""Kill the pty-backed channel (which kills the remote python
process with it) and tear down the SSH connection. Call once at
app shutdown."""
if self._channel:
try: self._channel.close()
except: pass
self._channel = None
if self._ssh_client:
try: self._ssh_client.close()
except: pass
self._ssh_client = None
self.ready = False
# ==========================================
# MAIN TELEOP
# ==========================================
class NaoTeleop:
def __init__(self, session, nao_ip, audio_device_index=None, mic_channel=1, mic_gain=8.0, video_scale=2.0,
nao_ssh_user="nao", nao_ssh_password="nao", nao_ssh_port=22, relay_port=5555, relay_rate=16000,
video_fps=25, media_relay=None):
video_fps=25, media_relay=None, deploy_video_server=True, video_server_path=None):
self.session = session
self.nao_ip = nao_ip
self.motion = session.service("ALMotion")
@@ -419,6 +546,17 @@ class NaoTeleop:
self._movement_stop_event = threading.Event()
self._movement_thread = None
# If we're using the on-robot media relay, deploy+launch
# nao_video_server.py over SSH now (before _init_video_maxfps
# tries to connect to it) instead of requiring it to already be
# running manually.
self.video_server_launcher = None
if self.media_relay and deploy_video_server:
self.video_server_launcher = VideoServerLauncher(
nao_ip, ssh_user=nao_ssh_user, ssh_password=nao_ssh_password,
ssh_port=nao_ssh_port, local_path=video_server_path)
self.video_server_launcher.deploy_and_start()
self._init_audio_device()
self._init_video_maxfps()
if HAS_AUDIO:
@@ -451,8 +589,8 @@ class NaoTeleop:
if self.media_relay:
self.video = None
self.video_client = "relay" # truthy sentinel, unsubscribe() is a no-op for this path
print(f"✅ Camera + mic via on-robot media relay: {self.media_relay}")
threading.Thread(target=self._media_relay_loop, daemon=True).start()
print(f"✅ Camera via on-robot media relay: {self.media_relay}")
threading.Thread(target=self._video_relay_loop, daemon=True).start()
return
try:
self.video = self.session.service("ALVideoDevice")
@@ -482,13 +620,13 @@ class NaoTeleop:
except Exception:
time.sleep(0.05)
def _media_relay_loop(self):
def _video_relay_loop(self):
"""Same role as _video_loop, but speaks the lean length-framed
TCP protocol from nao_video_server.py instead of pulling an
HTTP MJPEG stream - and now demuxes BOTH video and mu-law-
compressed mic audio off the same connection, since the mic
subscription lives on the robot too now instead of shipping
raw PCM through the qi RPC session."""
TCP protocol from nao_video_server.py instead of pulling raw
frames over the qi session. Video-only connection - it used to
share one socket with audio, but a slow video send could then
block audio behind it, so they're split (see the audio port,
media_relay port + 1, handled by _audio_relay_loop)."""
host, _, port_str = self.media_relay.partition(":")
port = int(port_str) if port_str else 8000
while not self._video_stop_event.is_set():
@@ -497,24 +635,50 @@ class NaoTeleop:
sock = socket.create_connection((host, port), timeout=5)
sock.settimeout(5.0)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
print(f"✅ Media relay connected: {host}:{port}")
print(f"✅ Video relay connected: {host}:{port}")
while not self._video_stop_event.is_set():
msg_type, length = struct.unpack(">BI", _recv_exact(sock, 5))
payload = _recv_exact(sock, length) if length else b""
if msg_type == TYPE_VIDEO:
arr = np.frombuffer(payload, dtype=np.uint8)
frame = cv2.imdecode(arr, cv2.IMREAD_COLOR)
if frame is not None:
self._latest_frame = cv2.resize(frame, (320, 240), interpolation=cv2.INTER_NEAREST)
self._latest_frame_ts = time.time()
elif msg_type == TYPE_AUDIO:
if getattr(self, "sound_receiver", None):
pcm = decode_ulaw(np.frombuffer(payload, dtype=np.uint8))
self.sound_receiver.feed(pcm.tobytes())
# TYPE_HELLO carries {"rate", "channels"} for future use - nothing to do with it yet.
# TYPE_HELLO: nothing to do with it yet, just skip.
except Exception as e:
print(f"media relay error: {e}")
print(f"video relay error: {e}")
finally:
if sock:
try: sock.close()
except: pass
if not self._video_stop_event.is_set():
time.sleep(0.5)
def _audio_relay_loop(self):
"""Mic audio's own connection - see _video_relay_loop for why
this is split off rather than sharing video's socket. Connects
to media_relay's host:port+1 by convention (matches AUDIO_PORT
= VIDEO_PORT + 1 in nao_video_server.py)."""
host, _, port_str = self.media_relay.partition(":")
video_port = int(port_str) if port_str else 8000
port = video_port + 1
while not self._video_stop_event.is_set():
sock = None
try:
sock = socket.create_connection((host, port), timeout=5)
sock.settimeout(5.0)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
print(f"✅ Audio relay connected: {host}:{port}")
while not self._video_stop_event.is_set():
msg_type, length = struct.unpack(">BI", _recv_exact(sock, 5))
payload = _recv_exact(sock, length) if length else b""
if msg_type == TYPE_AUDIO and getattr(self, "sound_receiver", None):
pcm = decode_ulaw(np.frombuffer(payload, dtype=np.uint8))
self.sound_receiver.feed(pcm.tobytes())
# TYPE_HELLO: nothing to do with it yet, just skip.
except Exception as e:
print(f"audio relay error: {e}")
finally:
if sock:
try: sock.close()
@@ -533,11 +697,12 @@ class NaoTeleop:
try:
# Playback machinery is needed either way - only the source
# of audio chunks differs (qi pub/sub below, vs mu-law
# chunks arriving over the media relay in _media_relay_loop).
# chunks arriving over the audio relay connection).
self.sound_receiver = SoundReceiver(self.audio_device_index, self.mic_gain,
pyaudio_instance=self._pyaudio)
if self.media_relay:
print(f"✅ Mic stream ready (gain={self.mic_gain}x, via media relay - compressed)")
print(f"✅ Mic stream ready (gain={self.mic_gain}x, via media relay - compressed, own connection)")
threading.Thread(target=self._audio_relay_loop, daemon=True).start()
return
self.session.registerService(self.audio_service_name, self.sound_receiver)
time.sleep(0.5)
@@ -970,6 +1135,8 @@ class NaoTeleop:
if getattr(self, "mic_relay", None):
self._safe("mic relay stop", self.mic_relay.stop)
self._safe("mic relay close", self.mic_relay.close)
if getattr(self, "video_server_launcher", None):
self._safe("video server stop", self.video_server_launcher.close)
if self.audio_device and self._qi_audio_subscribed:
self._safe("audio unsubscribe", lambda: self.audio_device.unsubscribe(self.audio_service_name))
if getattr(self, "sound_receiver", None):
@@ -998,8 +1165,11 @@ if __name__ == "__main__":
"(e.g. 10-12) when tunneling over a constrained link. "
"Ignored when --media-relay is set (the server controls fps).")
parser.add_argument("--media-relay", type=str, default=None,
help="host:port of the on-robot media relay (nao_video_server.py), "
"e.g. 127.0.0.1:8000 when tunneled through the phone. When set, "
help="host:port to reach the on-robot media relay (nao_video_server.py) "
"at, e.g. 127.0.0.1:8000 when tunneled through the phone. Setting "
"this also makes naowalk auto-scp nao_video_server.py to the "
"robot's home dir over SSH and launch it (see --no-deploy-video-"
"server to skip that and connect to one you started yourself). "
"BOTH video and mic audio come compressed over this one TCP "
"connection (JPEG + mu-law) instead of raw frames/PCM over the "
"qi session - use this on a tunneled/shoddy link. --mic-channel "
@@ -1016,6 +1186,13 @@ if __name__ == "__main__":
help="TCP port used to stream mic audio to the robot's nc listener")
parser.add_argument("--relay-rate", type=int, default=16000,
help="Sample rate for the live mic relay - try 48000 if audio sounds sped up/slow")
parser.add_argument("--no-deploy-video-server", action="store_true",
help="Don't auto-scp/launch nao_video_server.py on the robot when "
"--media-relay is set - use this if you're already running it "
"yourself and just want naowalk to connect to it.")
parser.add_argument("--video-server-path", type=str, default=None,
help="Local path to nao_video_server.py to deploy (default: the "
"copy sitting next to naowalk.py)")
args = parser.parse_args()
session = qi.Session()
@@ -1037,4 +1214,6 @@ if __name__ == "__main__":
nao_ssh_password=args.nao_ssh_password,
nao_ssh_port=args.nao_ssh_port,
relay_port=args.relay_port,
relay_rate=args.relay_rate).run()
relay_rate=args.relay_rate,
deploy_video_server=not args.no_deploy_video_server,
video_server_path=args.video_server_path).run()