Add hermes_audio and draw the mouth from the real waveform

audio_state taps the microphone and the speaker monitor with parec and publishes
levels, a speaking/listening/quiet activity, and trigger-locked waveform windows;
audio itself stays out of ROS. The activity detector calibrates its noise floor
at start-up and absorbs steady noise, so a room at -43 dBFS does not read as
"listening" forever, and it ignores the mic while the robot speaks.

The face arbitrates the state it shows: sleeping (agent side only, so room sound
never wakes a face that means "no agent connected") > speaking > listening >
thinking > idle, and draws the mouth from the real samples while fresh scope data
exists, auto-scaled so a quiet voice is still visible.

Checked: a tone played to the sink shows as `speaking` with the mic ignored. The
microphone-to-mouth path with a live voice is not yet confirmed.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AFN8nRRCiDiMN7ChBpHJhs

ros/hermes_face/hermes_face/arbiter.py

@@ -0,0 +1,22 @@
+"""Which state the face shows, from what the agent says and what the microphone/speaker hear.
+
+Priority, highest first: sleeping > speaking > listening > thinking > idle. `sleeping` means "no
+agent connected", so only the agent side can set or clear it: room sound never wakes the face, or it
+would claim the agent hears you. `looking` is a timed event the renderer owns, not decided here.
+No ROS imports.
+"""
+from __future__ import annotations
+
+
+def resolve_state(agent: str, audio: str) -> str:
+    """agent: the last /hermes/state (sleeping idle listening thinking speaking);
+    audio: /audio/activity (speaking listening quiet)."""
+    if agent == 'sleeping':
+        return 'sleeping'
+    if audio == 'speaking' or agent == 'speaking':
+        return 'speaking'
+    if audio == 'listening' or agent == 'listening':
+        return 'listening'
+    if agent == 'thinking':
+        return 'thinking'
+    return 'idle'

ros/hermes_face/hermes_face/face_node.py

@@ -5,6 +5,10 @@ Subscribes:  /hermes/state (std_msgs/String): sleeping idle listening thinking l
                  convention is the MCP tool's: positive pan = the robot's left, positive tilt = up.
              /vision/faces (vision_msgs/Detection2DArray): the eyes glance toward the best face
              /camera/camera_info (sensor_msgs/CameraInfo): the image size for that, if published
+             /audio/activity (std_msgs/String): speaking | listening | quiet, from hermes_audio
+             /audio/scope/{mic,speaker} (std_msgs/Float32MultiArray): the real waveform for the mouth
+Which state shows is decided by hermes_face.arbiter: sleeping (agent only) > speaking > listening >
+thinking > idle; room sound never wakes a sleeping face.
 The face starts in `initial_state` (default `sleeping`), so it runs with nothing else up; whichever
 node knows the agent is connected (hermes_mcp) publishes `idle` to wake it.
 """
@@ -14,9 +18,10 @@ import time
 import rclpy
 from rclpy.node import Node
 from sensor_msgs.msg import CameraInfo, JointState
-from std_msgs.msg import String
+from std_msgs.msg import Float32MultiArray, String
 from vision_msgs.msg import Detection2DArray
 
+from .arbiter import resolve_state
 from .gaze import Target, gaze_from_pixel, pick_target
 from .render import STATES, FaceRenderer
 
@@ -30,6 +35,9 @@ class FaceNode(Node):
         self.pan_joint = param('pan_joint', 'pan_joint')
         self.tilt_joint = param('tilt_joint', 'tilt_joint')
         initial = param('initial_state', 'sleeping')
+        self.agent_state = initial if initial in STATES else 'sleeping'
+        self.audio_activity = 'quiet'
+        self.audio_drive = bool(param('audio_drive', True))
         self.follow = bool(param('gaze_follow', True))
         self.image_size = (float(param('camera_width', 1296)), float(param('camera_height', 972)))
 
@@ -49,6 +57,11 @@ class FaceNode(Node):
 
         self.create_subscription(String, '/hermes/state', self._on_state, 10)
         self.create_subscription(JointState, '/joint_states', self._on_joints, 10)
+        if self.audio_drive:
+            self.create_subscription(String, '/audio/activity', self._on_activity, 10)
+            for kind in ('mic', 'speaker'):
+                self.create_subscription(Float32MultiArray, f'/audio/scope/{kind}',
+                                         lambda m, k=kind: self.renderer.set_scope(k, m.data, time.monotonic()), 2)
         if self.follow:
             self.create_subscription(Detection2DArray, '/vision/faces', self._on_faces, 10)
             self.create_subscription(CameraInfo, '/camera/camera_info', self._on_camera_info, 1)
@@ -61,7 +74,20 @@ class FaceNode(Node):
         if msg.data not in STATES:
             self.get_logger().warn(f'ignoring unknown state {msg.data!r}; known: {", ".join(STATES)}')
             return
-        self.renderer.set_state(msg.data, time.monotonic())
+        now = time.monotonic()
+        if msg.data == 'looking':          # a timed event: the renderer squints, flashes, then goes back
+            self.renderer.set_state('looking', now)
+            return
+        self.agent_state = msg.data
+        self._resolve(now)
+
+    def _on_activity(self, msg: String) -> None:
+        self.audio_activity = msg.data
+        self._resolve(time.monotonic())
+
+    def _resolve(self, now: float) -> None:
+        if self.renderer.state != 'looking':
+            self.renderer.set_state(resolve_state(self.agent_state, self.audio_activity if self.audio_drive else 'quiet'), now)
 
     def _on_joints(self, msg: JointState) -> None:
         pos = dict(zip(msg.name, msg.position))
@@ -84,6 +110,7 @@ class FaceNode(Node):
 
     def _tick(self) -> None:
         now = time.monotonic()
+        self._resolve(now)                 # also settles the state a finished `looking` returned to
         rgb = self.renderer.frame(now, now - self._last, 'rgb565')
         self._last = now
         if self.display:

ros/hermes_face/hermes_face/render.py

@@ -30,6 +30,7 @@ LOOK_HOLD_S = 1.2            # how long `looking` holds before returning to the
 SQUINT_S = 0.18              # squint before the shutter flash
 FLASH_AT_S = 0.16            # the shutter bloom lands this long after entering `looking`
 SLEEP_OPEN = 0.07            # eye openness while asleep: a thin line
+SCOPE_FRESH_S = 0.35         # a waveform window older than this is ignored (the synthetic voice returns)
 GAZE_HOLD_S = 2.5            # a glance toward a detected person lasts this long without a refresh
 GAZE_RANGE = (12.0, 9.0)     # eye travel in pixels for a gaze of +-1 (display x, y)
 
@@ -152,6 +153,8 @@ class FaceRenderer:
         self._flash_pending = state == 'looking'
         self.pan = 0.0     # degrees, from the servos: positive pan = robot's left, tilt = looks up
         self.tilt = 0.0
+        self._scope = {'mic': (None, -1e9), 'speaker': (None, -1e9)}   # newest real waveform per source
+        self._scope_peak = {'mic': 0.05, 'speaker': 0.05}
         self._gaze = (0.0, 0.0)      # where a person is, as eye direction: +x viewer's right, +y down
         self._gaze_t = -1e9
         self.a = dict(think=0.0, open=SLEEP_OPEN if state == 'sleeping' else 1.0, eye_scale=1.0,
@@ -192,6 +195,25 @@ class FaceRenderer:
     def set_pose(self, pan_deg: float, tilt_deg: float) -> None:
         self.pan, self.tilt = pan_deg, tilt_deg
 
+    def set_scope(self, kind: str, samples, now: float) -> None:
+        """The real waveform for the mouth: a trigger-locked window of samples in [-1, 1] from the
+        'mic' or the 'speaker'. While it is fresh and the face is listening/speaking, the mouth draws
+        it instead of the synthetic voice, auto-scaled so a quiet voice is still visible."""
+        if kind not in self._scope:
+            raise ValueError(f'unknown scope {kind!r}')
+        x = np.asarray(samples, np.float32)
+        if len(x) < 8:
+            return
+        peak = float(np.abs(x).max())
+        self._scope_peak[kind] = max(peak, self._scope_peak[kind] * 0.97, 0.02)   # fast attack, slow release
+        self._scope[kind] = (x, now)
+
+    def _real_wave(self, kind: str, T: float):
+        x, t = self._scope[kind]
+        if x is None or T - t > SCOPE_FRESH_S:
+            return None
+        return x / self._scope_peak[kind]
+
     def set_gaze(self, gx: float, gy: float, now: float) -> None:
         """Glance toward a person: gx, gy in [-1, 1] as eye direction (+x is the viewer's right, +y
         down). Held for GAZE_HOLD_S, then the eyes drift back to centre unless it is refreshed."""
@@ -297,16 +319,26 @@ class FaceRenderer:
         mouth_w = (mx1 - mx0) * (1 - 0.45 * k) * (1 - 0.5 * sleep)
         m_start = (mx0 + mx1) / 2 - mouth_w / 2
 
+        real_s = self._real_wave('speaker', T) if self.state == 'speaking' else None
+        real_l = self._real_wave('mic', T) if self.state == 'listening' else None
+
+        def real(wave, u):
+            return np.interp(u * (len(wave) - 1), np.arange(len(wave)), wave)
+
         def mouth(u):
             x = m_start + u * mouth_w
             c = 2 * u - 1
             y = my + smile * (1 - c * c) - smile * 0.4
-            if speak_amp > 0.05:
+            if real_s is not None:                   # the actual outgoing audio
+                y = y - 26 * a['mouth_speak'] * real(real_s, u)
+            elif speak_amp > 0.05:
                 y = y - speak_amp * voice_sample(sv, u * win_s, 0.0)
                 if sv.noise:
                     y = y - sv.noise * 5 * a['mouth_speak'] * (np.sin(u * 97 + T * 53) *
                                                                np.sin(u * 61 - T * 31))
-            if listen_amp > 0.05:
+            if real_l is not None:                   # the actual incoming audio, dimmer and smaller
+                y = y - 12 * a['mouth_listen'] * real(real_l, u)
+            elif listen_amp > 0.05:
                 y = y - listen_amp * voice_sample(lv, u * win_l, 0.4)
             return x, y
         self._trace(mouth, 320, e * 0.65 * (0.95 - 0.3 * a['mouth_listen']))

ros/hermes_face/test/test_arbiter.py

@@ -0,0 +1,20 @@
+import pytest
+
+from hermes_face.arbiter import resolve_state
+
+
+@pytest.mark.parametrize('agent,audio,expected', [
+    ('sleeping', 'quiet', 'sleeping'),
+    ('sleeping', 'listening', 'sleeping'),       # room sound never wakes the face
+    ('sleeping', 'speaking', 'sleeping'),
+    ('idle', 'quiet', 'idle'),
+    ('idle', 'listening', 'listening'),
+    ('idle', 'speaking', 'speaking'),
+    ('thinking', 'quiet', 'thinking'),
+    ('thinking', 'listening', 'listening'),      # a voice interrupts the thinking face
+    ('listening', 'quiet', 'listening'),
+    ('speaking', 'quiet', 'speaking'),           # the agent says it is speaking before the audio shows it
+    ('thinking', 'speaking', 'speaking'),
+])
+def test_priority(agent, audio, expected):
+    assert resolve_state(agent, audio) == expected

ros/hermes_face/test/test_render.py

@@ -144,3 +144,41 @@ def test_gaze_is_clamped():
     r = FaceRenderer(seed=0)
     r.set_gaze(50, -50, 0.0)
     assert r._gaze == (1.0, -1.0)
+
+
+def mouth_rows(frame):
+    """Row extent of lit pixels in the mouth band."""
+    band = frame[140:215].astype(np.float32).sum(axis=(1, 2))
+    rows = np.nonzero(band > 0.15 * band.max())[0]
+    return int(rows.max() - rows.min()) if len(rows) else 0
+
+
+def run_scope(kind, state, amp):
+    r = FaceRenderer(state=state, seed=0, jitter=0)
+    t, frame = 0.0, None
+    for _ in range(60):
+        t += 1 / 30
+        r.set_scope(kind, amp * np.sin(np.linspace(0, 8 * np.pi, 256)), t)   # refreshed every frame
+        frame = r.frame(t, 1 / 30)
+    return frame
+
+
+def test_the_mouth_draws_the_real_waveform_and_autoscales_it():
+    big, small = run_scope('speaker', 'speaking', 0.8), run_scope('speaker', 'speaking', 0.02)
+    assert mouth_rows(big) > 25 and mouth_rows(small) > 25    # a quiet voice is still visible
+    assert abs(mouth_rows(big) - mouth_rows(small)) < 8
+
+
+def test_the_mic_waveform_is_drawn_only_while_listening_and_stale_data_is_ignored():
+    listening = mouth_rows(run_scope('mic', 'listening', 0.5))
+    idle = mouth_rows(run_scope('mic', 'idle', 0.5))
+    assert listening > idle + 6
+    r = FaceRenderer(state='speaking', seed=0, jitter=0)
+    r.set_scope('speaker', np.sin(np.linspace(0, 6, 128)), 0.0)
+    frame, _ = run(r, 2)                                      # the scope goes stale: synthetic voice, no crash
+    assert frame.max() > 100
+
+
+def test_unknown_scope_kind_rejected():
+    with pytest.raises(ValueError):
+        FaceRenderer().set_scope('radio', [0.0] * 16, 0.0)