Add hermes_audio and draw the mouth from the real waveform

audio_state taps the microphone and the speaker monitor with parec and publishes
levels, a speaking/listening/quiet activity, and trigger-locked waveform windows;
audio itself stays out of ROS. The activity detector calibrates its noise floor
at start-up and absorbs steady noise, so a room at -43 dBFS does not read as
"listening" forever, and it ignores the mic while the robot speaks.

The face arbitrates the state it shows: sleeping (agent side only, so room sound
never wakes a face that means "no agent connected") > speaking > listening >
thinking > idle, and draws the mouth from the real samples while fresh scope data
exists, auto-scaled so a quiet voice is still visible.

Checked: a tone played to the sink shows as `speaking` with the mic ignored. The
microphone-to-mouth path with a live voice is not yet confirmed.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AFN8nRRCiDiMN7ChBpHJhs
This commit is contained in:
2026-10-04 23:26:01 -07:00
co-authored by Claude Sonnet 5.5
parent 880bdd3f44
commit ac4e09bf35
15 changed files with 534 additions and 4 deletions
+38
View File
@@ -144,3 +144,41 @@ def test_gaze_is_clamped():
r = FaceRenderer(seed=0)
r.set_gaze(50, -50, 0.0)
assert r._gaze == (1.0, -1.0)
def mouth_rows(frame):
"""Row extent of lit pixels in the mouth band."""
band = frame[140:215].astype(np.float32).sum(axis=(1, 2))
rows = np.nonzero(band > 0.15 * band.max())[0]
return int(rows.max() - rows.min()) if len(rows) else 0
def run_scope(kind, state, amp):
r = FaceRenderer(state=state, seed=0, jitter=0)
t, frame = 0.0, None
for _ in range(60):
t += 1 / 30
r.set_scope(kind, amp * np.sin(np.linspace(0, 8 * np.pi, 256)), t) # refreshed every frame
frame = r.frame(t, 1 / 30)
return frame
def test_the_mouth_draws_the_real_waveform_and_autoscales_it():
big, small = run_scope('speaker', 'speaking', 0.8), run_scope('speaker', 'speaking', 0.02)
assert mouth_rows(big) > 25 and mouth_rows(small) > 25 # a quiet voice is still visible
assert abs(mouth_rows(big) - mouth_rows(small)) < 8
def test_the_mic_waveform_is_drawn_only_while_listening_and_stale_data_is_ignored():
listening = mouth_rows(run_scope('mic', 'listening', 0.5))
idle = mouth_rows(run_scope('mic', 'idle', 0.5))
assert listening > idle + 6
r = FaceRenderer(state='speaking', seed=0, jitter=0)
r.set_scope('speaker', np.sin(np.linspace(0, 6, 128)), 0.0)
frame, _ = run(r, 2) # the scope goes stale: synthetic voice, no crash
assert frame.max() > 100
def test_unknown_scope_kind_rejected():
with pytest.raises(ValueError):
FaceRenderer().set_scope('radio', [0.0] * 16, 0.0)