reduced latency by not having ssh run on every V press and just sending to tcp aplay when v is held
This commit is contained in:
Lucca Pirovano
+69
-33
@@ -152,10 +152,16 @@ class SoundReceiver:
|
||||
class LiveMicRelay:
|
||||
"""
|
||||
Push-to-talk: streams the local mic straight to NAO's own speaker.
|
||||
On start(), SSHes into the robot and launches `nc -l | aplay` there,
|
||||
then opens a plain TCP socket to that listener and pipes raw mic
|
||||
audio into it for as long as the key is held. Closing the socket on
|
||||
stop() ends the remote nc process, which ends aplay with it.
|
||||
|
||||
connect() (call once at startup, off the hot path) SSHes into the
|
||||
robot and launches a self-restarting `nc -l | aplay` loop that stays
|
||||
up for the whole session. Each start()/stop() (per push-to-talk
|
||||
press) then just opens/closes a plain TCP socket to that already-
|
||||
running listener and pipes raw mic audio into it - no SSH handshake
|
||||
per press. Closing the socket on stop() ends that iteration's nc,
|
||||
which ends aplay with it, and the remote loop immediately relaunches
|
||||
nc so it's ready for the next press. close() (call once at shutdown)
|
||||
tears down the persistent SSH connection and kills the remote loop.
|
||||
"""
|
||||
def __init__(self, nao_ip, ssh_user="nao", ssh_password="nao",
|
||||
port=5555, rate=16000, input_device_index=None, pyaudio_instance=None):
|
||||
@@ -167,6 +173,7 @@ class LiveMicRelay:
|
||||
self.input_device_index = input_device_index
|
||||
|
||||
self.active = False
|
||||
self.connected = False # remote listener is up, ready for fast local connects
|
||||
# "idle" | "connecting" | "live" - read by the HUD to show status
|
||||
self.state = "idle"
|
||||
self._ssh_client = None
|
||||
@@ -179,17 +186,62 @@ class LiveMicRelay:
|
||||
# backend (pa_atomic_load assertion).
|
||||
self._p = pyaudio_instance if pyaudio_instance is not None else (pyaudio.PyAudio() if HAS_AUDIO else None)
|
||||
|
||||
def connect(self):
|
||||
"""One-time setup: SSH in and launch a self-restarting remote
|
||||
listener (`nc | aplay` in a loop) that stays up for the whole app
|
||||
session. Call this once at startup, in a background thread so it
|
||||
doesn't block. After this, start()/stop() per push-to-talk press
|
||||
only do a fast local socket connect - no SSH in the hot path.
|
||||
"""
|
||||
if not HAS_SSH:
|
||||
print("🎙️ Live mic needs paramiko: pip install paramiko")
|
||||
return
|
||||
try:
|
||||
self._ssh_client = paramiko.SSHClient()
|
||||
self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
self._ssh_client.connect(self.nao_ip, username=self.ssh_user,
|
||||
password=self.ssh_password, timeout=5)
|
||||
self._ssh_channel = self._ssh_client.get_transport().open_session()
|
||||
# A pty ties the remote process group to this channel, so
|
||||
# closing it in close() reliably kills the loop/nc/aplay
|
||||
# instead of leaving an orphaned process on the robot.
|
||||
self._ssh_channel.get_pty()
|
||||
remote_cmd = (
|
||||
f"while true; do nc -l -p {self.port} | "
|
||||
f"aplay -q -r {self.rate} -f S16_LE -c 1 -t raw -; done"
|
||||
)
|
||||
self._ssh_channel.exec_command(remote_cmd)
|
||||
time.sleep(0.4) # one-time wait for the first nc to bind
|
||||
self.connected = True
|
||||
print("🎙️ Talk relay: remote listener ready")
|
||||
except Exception as e:
|
||||
print(f"🎙️ Talk relay: couldn't set up remote listener ({e})")
|
||||
self.connected = False
|
||||
|
||||
def close(self):
|
||||
"""Tear down the persistent SSH connection/listener. Call once at
|
||||
app shutdown - not per push-to-talk press."""
|
||||
if self._ssh_channel:
|
||||
try: self._ssh_channel.close()
|
||||
except: pass
|
||||
self._ssh_channel = None
|
||||
if self._ssh_client:
|
||||
try: self._ssh_client.close()
|
||||
except: pass
|
||||
self._ssh_client = None
|
||||
self.connected = False
|
||||
|
||||
def start(self):
|
||||
if self.active:
|
||||
return
|
||||
if not HAS_AUDIO:
|
||||
print("🎙️ Live mic needs pyaudio installed")
|
||||
return
|
||||
if not HAS_SSH:
|
||||
print("🎙️ Live mic needs paramiko: pip install paramiko")
|
||||
if not self.connected:
|
||||
print("🎙️ Talk relay: remote listener not ready yet")
|
||||
return
|
||||
# If a previous relay thread is still tearing itself down, wait for
|
||||
# it to finish before reusing self._stream/_sock/_ssh_* for a new one.
|
||||
# it to finish before reusing self._stream/_sock for a new one.
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=2.0)
|
||||
self.active = True
|
||||
@@ -200,7 +252,7 @@ class LiveMicRelay:
|
||||
def stop(self):
|
||||
"""Signal the relay thread to stop and wait for it to exit.
|
||||
|
||||
Deliberately does NOT touch self._stream/_sock/_ssh_* here: this is
|
||||
Deliberately does NOT touch self._stream/_sock here: this is
|
||||
called from the main/render thread on key-up, and closing a
|
||||
PortAudio stream from a different thread while the relay thread is
|
||||
mid blocking-read() on it is what caused the ALSA segfault on
|
||||
@@ -212,6 +264,9 @@ class LiveMicRelay:
|
||||
self._thread.join(timeout=2.0)
|
||||
|
||||
def _cleanup(self):
|
||||
"""Per-press teardown: only the local socket/mic stream. The SSH
|
||||
connection is persistent now - close() (at app shutdown) is what
|
||||
tears that down, not this."""
|
||||
if self._sock:
|
||||
try: self._sock.close()
|
||||
except: pass
|
||||
@@ -222,34 +277,11 @@ class LiveMicRelay:
|
||||
self._stream.close()
|
||||
except: pass
|
||||
self._stream = None
|
||||
if self._ssh_channel:
|
||||
try: self._ssh_channel.close()
|
||||
except: pass
|
||||
self._ssh_channel = None
|
||||
if self._ssh_client:
|
||||
try: self._ssh_client.close()
|
||||
except: pass
|
||||
self._ssh_client = None
|
||||
|
||||
def _run(self):
|
||||
try:
|
||||
self._ssh_client = paramiko.SSHClient()
|
||||
self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
self._ssh_client.connect(self.nao_ip, username=self.ssh_user,
|
||||
password=self.ssh_password, timeout=5)
|
||||
|
||||
# Raw 16-bit mono PCM in, straight to ALSA out. If pitch/speed
|
||||
# sounds off, the robot's ALSA default rate doesn't match
|
||||
# self.rate - pass --relay-rate to match it (try 48000).
|
||||
remote_cmd = (
|
||||
f"nc -l -p {self.port} | "
|
||||
f"aplay -q -r {self.rate} -f S16_LE -c 1 -t raw -"
|
||||
)
|
||||
print("🎙️ Talk relay: starting remote listener...")
|
||||
self._ssh_channel = self._ssh_client.get_transport().open_session()
|
||||
self._ssh_channel.exec_command(remote_cmd)
|
||||
time.sleep(0.4) # give nc a moment to bind before we connect
|
||||
|
||||
# Remote listener is already up (from connect()) - just make a
|
||||
# fast local socket connection, no SSH handshake here.
|
||||
self._sock = socket.create_connection((self.nao_ip, self.port), timeout=5)
|
||||
|
||||
self._stream = self._p.open(format=pyaudio.paInt16, channels=1,
|
||||
@@ -535,6 +567,9 @@ class NaoTeleop:
|
||||
self._safe("moveInit", self.motion.moveInit)
|
||||
print("🦶 Walk engine ready")
|
||||
|
||||
if getattr(self, "mic_relay", None):
|
||||
threading.Thread(target=self.mic_relay.connect, daemon=True).start()
|
||||
|
||||
print("\n🎮 Controls: WASD/Arrows=Walk | [/]=Speed | IJKL=Head | SPACE=Reset head | 1=Wave | 2=Cena | 3=6-7 | M=Music search | X=Stop | -/==Volume | T=TTS | V=Push-to-talk (hold) | ESC=Quit")
|
||||
|
||||
try:
|
||||
@@ -781,6 +816,7 @@ class NaoTeleop:
|
||||
self._safe("music stop", self.music.stop)
|
||||
if getattr(self, "mic_relay", None):
|
||||
self._safe("mic relay stop", self.mic_relay.stop)
|
||||
self._safe("mic relay close", self.mic_relay.close)
|
||||
if self.audio_device:
|
||||
self._safe("audio unsubscribe", lambda: self.audio_device.unsubscribe(self.audio_service_name))
|
||||
if getattr(self, "sound_receiver", None):
|
||||
|
||||
Reference in New Issue
Block a user