Lucca Pirovano
274 lines
10 KiB
Python
274 lines
10 KiB
Python
#!/usr/bin/env python
|
|
# -*- encoding: UTF-8 -*-
|
|
|
|
import qi
|
|
import argparse
|
|
import sys
|
|
import time
|
|
import threading
|
|
import queue
|
|
import cv2
|
|
import numpy as np
|
|
import pygame
|
|
|
|
# Import the new music module
|
|
from naomusic import NaoMusicPlayer
|
|
|
|
# --- Audio for mic ---
|
|
try:
|
|
import pyaudio
|
|
HAS_AUDIO = True
|
|
except ImportError:
|
|
HAS_AUDIO = False
|
|
|
|
# ==========================================
|
|
# NAO MIC STREAM
|
|
# ==========================================
|
|
class SoundReceiver:
|
|
PREBUFFER_CHUNKS = 3
|
|
SILENCE_CHUNK = b"\x00" * 4096
|
|
|
|
def __init__(self, output_device_index=None, mic_gain=8.0):
|
|
self.running = True
|
|
self.underrun_count = 0
|
|
self.chunks_received = 0
|
|
self._write_errors = 0
|
|
self._peak_since_print = 0
|
|
self._last_status_print = 0
|
|
self.mic_gain = float(mic_gain)
|
|
|
|
if HAS_AUDIO:
|
|
self.p = pyaudio.PyAudio()
|
|
print("🔈 Available output devices:")
|
|
for i in range(self.p.get_device_count()):
|
|
info = self.p.get_device_info_by_index(i)
|
|
if info.get("maxOutputChannels", 0) > 0:
|
|
print(f" [{i}] {info['name']}")
|
|
try:
|
|
chosen = (self.p.get_device_info_by_index(output_device_index) if output_device_index is not None
|
|
else self.p.get_default_output_device_info())
|
|
print(f"🔈 Using: [{chosen['index']}] {chosen['name']}")
|
|
except: pass
|
|
|
|
self.stream = self.p.open(format=pyaudio.paInt16, channels=1, rate=16000,
|
|
output=True, output_device_index=output_device_index,
|
|
frames_per_buffer=2048)
|
|
self._play_test_tone()
|
|
self.queue = queue.Queue(maxsize=40)
|
|
self.playback_thread = threading.Thread(target=self._playback_loop, daemon=True)
|
|
self.playback_thread.start()
|
|
|
|
def _play_test_tone(self):
|
|
try:
|
|
duration, freq, rate = 0.3, 440.0, 16000
|
|
t = np.linspace(0, duration, int(rate * duration), False)
|
|
tone = (np.sin(freq * t * 2 * np.pi) * 12000).astype(np.int16)
|
|
self.stream.write(tone.tobytes())
|
|
except: pass
|
|
|
|
def processRemote(self, nbOfChannels, nbOfSamplesByChannel, timeStamp, buffer):
|
|
if not HAS_AUDIO: return
|
|
self.chunks_received += 1
|
|
try:
|
|
self.queue.put_nowait(bytes(buffer))
|
|
except queue.Full:
|
|
try:
|
|
self.queue.get_nowait()
|
|
self.queue.put_nowait(bytes(buffer))
|
|
except: pass
|
|
|
|
def _playback_loop(self):
|
|
primed = []
|
|
while self.running and len(primed) < self.PREBUFFER_CHUNKS:
|
|
try:
|
|
primed.append(self.queue.get(timeout=1.0))
|
|
except queue.Empty:
|
|
break
|
|
for chunk in primed:
|
|
self._write_chunk(chunk)
|
|
|
|
while self.running:
|
|
try:
|
|
chunk = self.queue.get(timeout=0.2)
|
|
self._write_chunk(chunk)
|
|
except queue.Empty:
|
|
self.underrun_count += 1
|
|
try:
|
|
self.stream.write(self.SILENCE_CHUNK)
|
|
except: pass
|
|
|
|
now = time.time()
|
|
if now - self._last_status_print > 10:
|
|
self._last_status_print = now
|
|
print(f"🎤 mic: {self.chunks_received} chunks, {self.underrun_count} underruns, peak {self._peak_since_print}")
|
|
self._peak_since_print = 0
|
|
|
|
def _write_chunk(self, raw_bytes):
|
|
try:
|
|
samples = np.frombuffer(raw_bytes, dtype=np.int16).astype(np.float32)
|
|
if samples.size:
|
|
amplified = np.clip(samples * self.mic_gain, -32768, 32767)
|
|
peak = int(np.abs(amplified).max())
|
|
if peak > self._peak_since_print:
|
|
self._peak_since_print = peak
|
|
self.stream.write(amplified.astype(np.int16).tobytes())
|
|
else:
|
|
self.stream.write(raw_bytes)
|
|
except: pass
|
|
|
|
def close(self):
|
|
self.running = False
|
|
if HAS_AUDIO:
|
|
try: self.playback_thread.join(timeout=1.0)
|
|
except: pass
|
|
try:
|
|
self.stream.stop_stream()
|
|
self.stream.close()
|
|
self.p.terminate()
|
|
except: pass
|
|
|
|
# ==========================================
|
|
# MAIN TELEOP
|
|
# ==========================================
|
|
class NaoTeleop:
|
|
def __init__(self, session, nao_ip, audio_device_index=None, mic_channel=1, mic_gain=8.0, video_scale=2.0):
|
|
self.session = session
|
|
self.nao_ip = nao_ip
|
|
self.motion = session.service("ALMotion")
|
|
self.posture = session.service("ALRobotPosture")
|
|
self.tts = session.service("ALTextToSpeech")
|
|
try:
|
|
self.battery = session.service("ALBattery")
|
|
except:
|
|
self.battery = None
|
|
|
|
self.music = NaoMusicPlayer(session, nao_ip)
|
|
self.music_results = []
|
|
self.selected_result = 0
|
|
|
|
self.audio_device = None
|
|
self.audio_service_name = "SoundReceiver"
|
|
self.audio_device_index = audio_device_index
|
|
self.mic_channel = mic_channel
|
|
self.mic_gain = mic_gain
|
|
|
|
self.running = True
|
|
self.head_yaw = 0.0
|
|
self.head_pitch = 0.0
|
|
self.battery_level = 100
|
|
self.last_batt_check = 0
|
|
self.volume = 50
|
|
self.walk_speed = 0.6 # Default walking speed
|
|
|
|
self.typing_mode = False
|
|
self.chat_message = ""
|
|
self.music_search_mode = False
|
|
self.music_search_text = ""
|
|
self.music_search_pending = False
|
|
|
|
self.video_scale = video_scale
|
|
self.display_width = int(320 * video_scale)
|
|
self.display_height = int(240 * video_scale)
|
|
|
|
self._init_audio_device()
|
|
self._init_video_maxfps()
|
|
if HAS_AUDIO:
|
|
self._init_mic_stream()
|
|
|
|
pygame.init()
|
|
self.screen = pygame.display.set_mode((self.display_width, self.display_height))
|
|
pygame.display.set_caption("NAO Teleop + Music on Robot")
|
|
self.font = pygame.font.SysFont(None, 24)
|
|
self.clock = pygame.time.Clock()
|
|
self.update_title()
|
|
|
|
def _safe(self, label, fn):
|
|
"""Run fn(), log+swallow any exception so one failure can't crash the loop."""
|
|
try:
|
|
return fn()
|
|
except Exception as e:
|
|
print(f"{label} error: {e}")
|
|
return None
|
|
|
|
def _init_audio_device(self):
|
|
try:
|
|
self.audio_device = self.session.service("ALAudioDevice")
|
|
self.volume = self.audio_device.getOutputVolume()
|
|
print(f"🔊 NAO speaker volume: {self.volume}%")
|
|
except Exception as e:
|
|
print(f"⚠️ ALAudioDevice error: {e}")
|
|
self.audio_device = None
|
|
|
|
def _init_video_maxfps(self):
|
|
try:
|
|
self.video = self.session.service("ALVideoDevice")
|
|
self.video_client = self.video.subscribeCamera("TeleopCam", 0, 0, 11, 25)
|
|
print("✅ Camera ready")
|
|
except:
|
|
print("❌ Camera not available")
|
|
self.video_client = None
|
|
|
|
def _init_mic_stream(self):
|
|
if not self.audio_device: return
|
|
try:
|
|
self.sound_receiver = SoundReceiver(self.audio_device_index, self.mic_gain)
|
|
self.session.registerService(self.audio_service_name, self.sound_receiver)
|
|
time.sleep(0.5)
|
|
self.audio_device.setClientPreferences(self.audio_service_name, 16000, self.mic_channel, 0)
|
|
self.audio_device.subscribe(self.audio_service_name)
|
|
print(f"✅ Mic stream ready (gain={self.mic_gain}x)")
|
|
except Exception as e:
|
|
print(f"❌ Mic init failed: {e}")
|
|
|
|
def get_image(self):
|
|
if not hasattr(self, 'video') or not self.video_client:
|
|
return None
|
|
try:
|
|
image = self.video.getImageRemote(self.video_client)
|
|
if image and len(image) >= 7:
|
|
w, h = image[0], image[1]
|
|
arr = np.frombuffer(bytearray(image[6]), dtype=np.uint8).reshape((h, w, 3))
|
|
frame = cv2.cvtColor(arr, cv2.COLOR_RGB2BGR)
|
|
return cv2.resize(frame, (320, 240), interpolation=cv2.INTER_NEAREST)
|
|
except:
|
|
pass
|
|
return None
|
|
|
|
def wave_gesture(self):
|
|
if self.typing_mode: return
|
|
print("🤚 Waving hello...")
|
|
try:
|
|
self.motion.setStiffnesses("RArm", 1.0)
|
|
names = ["RShoulderPitch", "RShoulderRoll", "RElbowYaw", "RElbowRoll", "RWristYaw", "RHand"]
|
|
angles = [[-1.5]*5, [-0.4,-0.7,-0.4,-0.7,-0.4], [0.6,1.1,0.6,1.1,0.6],
|
|
[0.8,0.3,0.8,0.3,0.8], [0.0,1.5,0.0,-1.5,0.0], [1.0]*5]
|
|
times = [[0.5,1.0,1.5,2.0,2.5]] * 6
|
|
self.motion.angleInterpolation(names, angles, times, True)
|
|
neutral = [1.5, -0.15, 1.2, 0.5, 0.0, 0.0]
|
|
self.motion.angleInterpolationWithSpeed(names, neutral, 0.3)
|
|
self.motion.setStiffnesses("RArm", 0.6)
|
|
except Exception as e:
|
|
print(f"Wave error: {e}")
|
|
|
|
def handle_head_movement(self):
|
|
if self.typing_mode or self.music_search_mode: return
|
|
keys = pygame.key.get_pressed()
|
|
speed = 0.3
|
|
changed = False
|
|
if keys[pygame.K_i]: self.head_pitch = max(self.head_pitch - speed, -0.5); changed = True
|
|
if keys[pygame.K_k]: self.head_pitch = min(self.head_pitch + speed, 0.5); changed = True
|
|
if keys[pygame.K_j]: self.head_yaw = min(self.head_yaw + speed, 2.0); changed = True
|
|
if keys[pygame.K_l]: self.head_yaw = max(self.head_yaw - speed, -2.0); changed = True
|
|
if changed:
|
|
self._safe("Head move", lambda: self.motion.setAngles(
|
|
["HeadYaw", "HeadPitch"], [self.head_yaw, self.head_pitch], 0.3))
|
|
|
|
def reset_head(self):
|
|
self.head_yaw = 0.0
|
|
self.head_pitch = 0.0
|
|
self._safe("Head reset", lambda: self.motion.setAngles(["HeadYaw", "HeadPitch"], [0.0, 0.0], 0.4))
|
|
|
|
def update_title(self):
|
|
mode = "[TYPING] " if self.typing_mode else ""
|
|
pygame.display.set_caption(f"NAO Teleop | {mode}Battery: {self.battery_level}% | Vol
|