From 720a7cfc349714977ae79bd0365043ac0c2496fa Mon Sep 17 00:00:00 2001 From: "K. Isom" Date: Mon, 5 Oct 2026 08:38:32 -0700 Subject: [PATCH] Bring every doc up to date, and give the repo a Wi-Fi template that exists CONTEXT.md had grown by appending and contradicted itself (all four nodes vs six, a retracted CPU figure, "no calibration", "nobody enrolled", "audio untested as a service", the Wi-Fi-off boot as not done, a class the tracker replaced). Rewritten as one current-state document organised by subsystem, with fresh measurements, the owner's decisions, the gotchas, the neck's arrival-day plan and what is genuinely still open. README: status, speakers not fitted, new-session note for Hermes, the calibration section (shipped calibration, coverage hints, outlier dropping, --from-views), running the tests, the docs row. HANDOFF.md gets a status banner listing where the original brief has drifted. CLAUDE.md starts at CONTEXT.md. The echo-cancel config no longer talks about a Pi Zero, and the enclosure guide no longer tells you to centre a servo with a tool that does not exist yet. pi/network-config.example is a tracked, credentials-free first-boot template: the README pointed at pi/50-cloud-init.yaml, which is gitignored and so missing from a fresh clone. The dead IdentityCache (replaced by the tracker) is removed. Docstrings corrected: audio scope publishing, the MCP health topics, and what the calibration tool does. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01AFN8nRRCiDiMN7ChBpHJhs --- CLAUDE.md | 7 +- CONTEXT.md | 653 ++++++++---------- HANDOFF.md | 17 + README.md | 47 +- enclosure/README.md | 5 +- host/omarchy/hermes-config-snippet.yaml | 5 +- pi/network-config.example | 24 + pi/pipewire/60-echo-cancel.conf.optional | 8 +- .../hermes_audio/audio_state_node.py | 5 +- ros/hermes_mcp/hermes_mcp/mcp_node.py | 3 +- ros/hermes_vision/hermes_vision/calibrate.py | 12 +- ros/hermes_vision/hermes_vision/recognize.py | 24 - ros/hermes_vision/test/test_identity_cache.py | 40 -- ros/hermes_vision/test/test_tracking.py | 8 + 14 files changed, 390 insertions(+), 468 deletions(-) create mode 100644 pi/network-config.example delete mode 100644 ros/hermes_vision/test/test_identity_cache.py diff --git a/CLAUDE.md b/CLAUDE.md index f59490e..3bb9e7c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,13 +5,14 @@ pan/tilt head and an oscilloscope-style face. Hardware: Raspberry Pi 4, Adafruit Arducam UC-346 (OV5647), two MG90S servos on GPIO12/13. Software: ROS 2 Jazzy on Ubuntu 24.04 on the Pi; on the hosts, ROS runs only in podman containers. -**Start with `HANDOFF.md`.** It is the implementation brief: hardware facts, what exists, the -work packages in order, acceptance checks, and the list of unverified assumptions. +**Start with `CONTEXT.md`**: where the work really stands, what is verified, the owner's decisions, and +what is open. `HANDOFF.md` is the original implementation brief (hardware facts, work packages, +acceptance checks); it has drifted, so read its status banner first. - `pi/` and `ros/` run on the robot; `host/` runs on the owner's machines. `CONTEXT.md` tracks where the work stands; `HANDOFF.md` is the original brief and has drifted (ROS stays on the Pi, and `hermes_mcp` is the only bridge out; see `CONTEXT.md`). -- Package 0a (platform proof) comes before everything; start with `host/check-pi-platform.sh`. +- The platform is proven (`host/check-pi-platform.sh` passes and the Pi runs everything); `CONTEXT.md` says what is open. - `docs/face-prototype.html` is the reference for the face renderer; port it, don't ship it. - `enclosure/` holds the OpenSCAD model and STLs. It was built from estimates; refine it from the owner's caliper measurements and test prints before anyone prints the large parts. diff --git a/CONTEXT.md b/CONTEXT.md index 11a5d46..f68c88e 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -1,401 +1,316 @@ # Current state -Last updated 2026-10-04. `HANDOFF.md` is the brief; this file tracks where the work stands. +Last updated 2026-10-05. This file says where the work really stands. `HANDOFF.md` is the original +brief and has drifted (see its banner); where they differ, this file wins. `README.md` is the guide for a +person using the robot. ## Hardware on hand -Raspberry Pi 4 (8 GB), BrainCraft HAT, Arducam UC-346 camera, and nothing else: **no servos, no -fan, no speakers**, no USB-C splitter, no printed enclosure (the camera hangs loose on its ribbon and is -held by hand). The Pi runs at about 62 C open-air with everything on and no fan; an enclosure will be -hotter, so fit the fan before enclosing it. Playback and the servos are untested. Work until those arrive is limited to what the Pi, HAT and camera can do. +Raspberry Pi 4 (8 GB), BrainCraft HAT, Arducam UC-346 camera, and **nothing else: no servos, no fan, no +speakers**, no USB-C splitter, no printed enclosure. The camera hangs loose on its ribbon and is held by +hand. The Pi is powered from talos's USB port (a bench setup, fine without servos). It runs at about +60 °C open-air with everything on; an enclosure will be hotter, so fit the fan before enclosing it. +Playback and the servos are untested because they cannot be. + +## What works (verified on the real robot) + +- **Face** on the display, started at power-up, with a `sleeping` pose that means "no agent connected". + The eyes glance toward detected faces; the mouth draws the live voice. Gaze direction and the voice + trace were confirmed by the owner on the real display. +- **Camera**, **face detection**, **opt-in recognition** (the owner is enrolled and recognised live). +- **MCP bridge**: `hermes mcp test robot` passes; real Hermes turns on the local `ornith` model call + `look()` and `faces()`. +- **USB network link**, **firewall**, and the whole robot **boots and works with Wi-Fi off**. +- **Thermal guard**, and a reboot-tested **single installer**, `pi/install.sh` (re-run on the live robot, + it changed nothing; **not yet run on a blank card**). +- 122 unit tests on the Pi: face 50, vision 30, audio 10, MCP 16, thermal 12, bringup 4. ## The Pi (`hermes-bot`) -- Ubuntu 24.04.5 server, kernel `6.8.0-1065-raspi` (fully upgraded), flashed from `talos`. User - `hermes` (key-only; passwordless sudo), SSH keys for `kyle@vade` (talos) and `kyle@straylight`. -- On Wi-Fi at `192.168.88.152` (MAC `d8:3a:dd:0d:66:f5`); `eth0` is not cabled. The owner says it has - a static IP on the rack switch; that address is not recorded here yet. **USB link: `10.55.0.2`** - (see package 0b below); `ssh hermes@10.55.0.2` works from talos. `avahi-daemon` is - installed, but mDNS is off in `systemd-resolved` on talos, so `hermes-bot.local` doesn't resolve - there; use the IP. -- **cloud-init applies `network-config` and `user-data` on the first boot only.** To change Wi-Fi - later, edit `/etc/netplan/50-cloud-init.yaml` on the Pi, or bump `instance_id` in `meta-data` on - `system-boot`. `pi/50-cloud-init.yaml` is a template for the Wi-Fi block; it is gitignored - because it holds credentials. -- Installed: ROS 2 Jazzy (`ros-base`, `ros-dev-tools`, colcon, `robot_state_publisher`, `xacro`, - `camera_ros`); `python3-numpy/pil/spidev/lgpio/rpi-lgpio`; a venv `~/venv` (system site packages - visible) with `adafruit-circuitpython-rgb-display` and Blinka. Workspace: `~/ros_ws`, source synced - from `ros/` with `rsync -a --delete --exclude __pycache__ ros/ hermes@:~/ros_ws/src/`, then - `colcon build --symlink-install`. -- **One unit runs the whole robot:** `robot.service` runs `ros2 launch hermes_bringup robot.launch.py` - as `hermes` (face, camera, face detection, audio levels), so everything is ROS and the face is up - whenever the robot is powered; each node is its own process, respawned by the launch file. Install - with `pi/install-robot-service.sh`. `RuntimeDirectory=robot` gives lgpio a writable working - directory (it drops a pipe in its cwd; systemd's default `/` is not writable). The face starts - first and the camera, vision and audio nodes follow `start_delay` (6 s) later, so the Python - processes do not fight over the CPU. Parameters are in `ros/hermes_bringup/config/robot.yaml`. - `robot-cpu-performance.service` holds the governor at `performance` (the face costs about half the - CPU at full clock). `pi/trim-boot.sh` disables snapd and ModemManager (unused; ~9 s of boot CPU). -- **Boot timing** (reboot-tested, everything returns by itself): the face draws at about 28 s on the - boot clock (kernel 6.9 s, `robot.service` at ~10 s into userspace, then `ros2 launch` takes ~10 s - to start on a cold Pi, then the face ~3 s); camera, audio and vision are up by ~37 s. The owner is - fine with up to a minute, and prefers everything in ROS to a faster non-ROS face; a split - `robot-face.service` that skipped the launch framework only saved 3.7 s and was dropped. -- **Steady-state CPU** with everything running (4 cores): face ~76%, camera ~57% (15 fps, from 30 - at ~40% alone), detector ~40% (was 131%), audio ~12%; about half the machine idle, 62 C, no - throttling, no fan fitted. Camera `FrameDurationLimits` is the lever (15 fps ~20% alone, 10 fps - ~14%); JPEG quality barely matters. OpenCV's DNN spins every core and burns 3-4x its elapsed - time in CPU, so the detector node pins it to one thread (YuNet 86 ms, SFace 161 ms of CPU), decodes - the JPEG at half size (`IMREAD_REDUCED_COLOR_2`, boxes scaled back to full-size pixels) and - re-identifies a tracked face at most once a second (`IdentityCache`). -- `ros2 node list` with its default short discovery window often shows only some of the nodes; use - `ros2 node list --no-daemon --spin-time 6` (all four appear), or check topics. -- **ROS never leaves the Pi** (owner's decision): `ROS_AUTOMATIC_DISCOVERY_RANGE=LOCALHOST` (set in - `pi/robot-env.sh`, which `source`s everything) and the firewall drops domain 42's DDS ports - (17900-18149) on every interface but loopback. `hermes_mcp` is the only bridge to the outside; a - host that wants data from the robot gets it through an MCP tool, not DDS. This supersedes - HANDOFF.md's `host/ros-shell` talking to the robot. (ROS's setup script exports `SUBNET`, so the - variable must be assigned, not defaulted.) -- **PipeWire** (1.0.5, WirePlumber 0.4.17) runs as `hermes`'s user services (`loginctl enable-linger`). - `~/.config/pipewire/pipewire-pulse.conf.d/50-network-server.conf` (from `pi/pipewire/`) makes - pipewire-pulse listen on TCP 4713 on all interfaces; the firewall is the protection. Defaults are - the WM8960 source and sink (`alsa_input/output.platform-soc_sound.stereo-fallback`). Verified from - talos: `pactl -s tcp:10.55.0.2:4713 info` works; the same on the Wi-Fi address times out. +- Ubuntu 24.04.5 server, kernel `6.8.0-1065-raspi`, flashed from talos. User `hermes` (key-only login, + passwordless sudo; no password is set). SSH keys for `kyle@vade` (talos) and `kyle@straylight`. +- Network: USB link `10.55.0.2` (host `10.55.0.1`); Wi-Fi `192.168.88.152` (MAC `d8:3a:dd:0d:66:f5`); + `eth0` is not cabled (the owner mentioned a static address on the rack switch; not recorded here). + `avahi-daemon` is installed, but mDNS is off in `systemd-resolved` on talos, so use the IP. `rfkill` + is installed. +- **cloud-init applies `network-config` and `user-data` on the first boot only.** To change Wi-Fi later, + edit `/etc/netplan/50-cloud-init.yaml` on the Pi, or bump `instance_id` in `meta-data` on + `system-boot`. `pi/network-config.example` is a credentials-free template for the Wi-Fi block (`pi/50-cloud-init.yaml` + is gitignored, for a filled copy). +- **Install:** `rsync` this repo to the Pi and `sudo ./pi/install.sh`. It installs ROS 2 Jazzy, builds + the Raspberry Pi libcamera fork (pinned commit) into `/opt/libcamera-rpi` and `camera_ros` against it + (`~/camera_ws`), makes `~/venv` (display libraries, OpenCV 4.10, the MCP SDK), downloads the face + models to `~/models`, restores the camera calibration, builds `~/ros_ws`, sets up the USB gadget, the + audio overlay and mixer, the firewall (before anything listens), PipeWire and the services, and + creates the MCP token. Update just the code with `rsync -a --delete --exclude __pycache__ ros/ + hermes@:~/ros_ws/src/`, `colcon build --symlink-install` in `~/ros_ws`, `sudo systemctl restart + robot`. Any shell that runs a node by hand must `source ~/robot-env.sh` first. +- **Services:** `robot.service` runs `ros2 launch hermes_bringup robot.launch.py` as `hermes`: the face, + camera, face detector, audio levels, MCP bridge and thermal guard, each its own process respawned by + the launch file. The face starts first; the others follow `start_delay` (6 s). Parameters: + `ros/hermes_bringup/config/robot.yaml`. `RuntimeDirectory=robot` gives lgpio a writable working + directory (it drops a pipe in its cwd; systemd's default `/` is not writable). Also + `robot-cpu-performance.service` (governor `performance`; the face costs about half the CPU at full + clock), `robot-firewall.service`, `robot-usb-gadget.service`. `pi/trim-boot.sh` disables snapd and + ModemManager (unused; ~9 s of boot CPU). +- **Boot:** the face draws about 28 s after the kernel starts (kernel ~7 s, `robot.service` ~10 s into + userspace, `ros2 launch` itself ~10 s on a cold Pi, the face ~3 s); the other nodes are up by ~37 s. + The owner is fine with up to a minute and prefers everything in ROS: a split face unit that skipped + `ros2 launch` saved only 3.7 s and was dropped. Reboot-tested: everything returns by itself. +- **CPU**, full stack, 4 cores: face ~84%, camera ~63% (15 fps; 30 fps alone is ~40%, 15 is ~20%), + detector ~45%, audio ~15%, MCP ~15%, PipeWire ~12%; about 40% of the machine idle, 60 °C, no + throttling. OpenCV's DNN spins every core and burns 3-4x its elapsed time in CPU, so the detector pins + it to one thread (YuNet 86 ms, SFace 161 ms of CPU), decodes the JPEG at half size and re-identifies a + tracked face at most once a second. +- **ROS never leaves the Pi** (owner's decision): `ROS_AUTOMATIC_DISCOVERY_RANGE=LOCALHOST` (ROS's + setup script exports `SUBNET`, so `pi/robot-env.sh` assigns it) and the firewall drops domain 42's DDS + ports (17900-18149) on every interface but loopback. `hermes_mcp` is the only bridge out. This + supersedes HANDOFF's `host/ros-shell`. +- **Firewall** (`pi/nftables-robot.nft`, loaded at boot before networking by `robot-firewall.service`): + accepts 4713 (audio), 8765 (MCP) and 8766 (reserved, nothing listens) only on `usb0` from `10.55.0.1` + and drops them elsewhere. SSH is deliberately open on all interfaces (key-only), for V2: the owner's call. +- **PipeWire** 1.0.5 / WirePlumber 0.4.17 run as `hermes`'s user services (`loginctl enable-linger`). + `pi/pipewire/50-network-server.conf` makes pipewire-pulse listen on TCP 4713 on all interfaces; the + firewall is the protection. Defaults are the WM8960 source and sink. -## Package 0a (platform proof on Ubuntu 24.04) +## Display and the face (`ros/hermes_face`) -- Step 1 done: `host/check-pi-platform.sh` passes. -- Step 2 done: flashed, booted, upgraded; ROS installed; `ros2 topic list` works. -- **Step 5 (display) done.** ST7789 works on `lgpio` through Blinka (`rpi-lgpio`, no `RPi.GPIO`). - Two gotchas: pass `cs=None` (the kernel drives CE0; claiming GPIO8 raises "GPIO busy"), and the - 240x240 glass sits at `y_offset=80` in the controller's RAM, for every rotation (the driver - rotates in software and leaves the scan direction fixed). Mounted pose (microSD end up, USB-C edge - on the left) needs content turned 90 degrees clockwise: `rotation=270`. Full-frame push is - ~23 fps through the driver; the face pushes raw RGB565 with `_block` and reaches ~27-28 fps. -- **Camera: works with the Raspberry Pi libcamera fork, not with the stock stacks.** The OV5647 is - detected (`ov5647 10-0036`) and Ubuntu's libcamera 0.2.0 lists it once `libcamera-ipa` is - installed, but capture dies in the Pi IPA (`assertion "it != buffers_.end()" in prepareIsp()`), - and ROS Jazzy's libcamera 0.7.2 dies with `FATAL … A list of V4L2 controls requires a - ControlInfoMap` right after `configure`. Both are deterministic whatever the stream format or - size. Fix: build the fork (`github.com/raspberrypi/libcamera`, commit `6c1dd9d`, 2026-08-17) with - `-Dpipelines=rpi/vc4 -Dipas=rpi/vc4 -Dcam=enabled -Dv4l2=false …` into `/opt/libcamera-rpi`; its - `cam` then captures 1296x972 RGB888 at ~16 fps (dim room). `pi/capture_still.py` takes a still - with it. **`camera_ros` still has to be built against this fork** (the apt one uses ROS's broken - libcamera). Frames are padded (stride 3936) and B,G,R in memory. -- **`camera_ros` 0.7.0 is built from source against the fork** in `~/camera_ws` (overlay on ROS; at - runtime the fork must come first in `LD_LIBRARY_PATH`, which `source ~/robot-env.sh` does; copy - from `pi/robot-env.sh`). Verified: `ros2 run camera_ros camera_node --ros-args -p width:=1296 -p - height:=972 -p format:=RGB888` publishes `/camera/image_raw`, `/camera/image_raw/compressed` (~26 - Hz) and `/camera/camera_info`. It costs ~140% of a core with raw and compressed both on; lower the - frame rate or size before running everything together. No calibration file yet. -- **`ros/hermes_vision`**: `hermes_vision.detect` (YuNet, Haar fallback; no ROS imports) and a node - `face_detector` that subscribes to the compressed image and publishes `/vision/faces` - (`vision_msgs/Detection2DArray`, bbox centre/size in pixels, `class_id: face`, score), at `rate_hz` - (default 4, ~47% of a core on a Pi 4); `publish_debug:=true` adds an annotated image. CLI: - `python -m hermes_vision.detect_cli IMAGE`. YuNet 2023mar needs OpenCV 4.8+, so `~/venv` has - `opencv-python-headless==4.10.0.84` with `numpy<2` (system NumPy stays 1.26.4 for ROS), and - `robot-env.sh` puts the venv first on `PYTHONPATH`; YuNet also needs input sides that are multiples - of 32. Tested on a sample image and on a live frame (one face found, upright, no flip needed). -- **Audio capture works (verified 2026-10-04).** The stock `wm8960-soundcard` overlay fails every - stream with `wm8960: No MCLK configured` on Ubuntu 24.04: it puts the 12.288 MHz fixed clock on the - sound card's codec link, but the codec driver reads `mclk` from its own node. - `pi/overlays/hermes-wm8960.dts` is the stock overlay plus `clocks`/`clock-names` on the codec node; - `pi/install-audio-overlay.sh` compiles it to `/boot/firmware/overlays/hermes-wm8960.dtbo`, sets - `dtparam=audio=off` and `dtoverlay=hermes-wm8960`. Capture then opens at 8-48 kHz. Address the - card by name (`hw:CARD=wm8960soundcard,DEV=0`): its index changes (it was card 1, then 0). -- **The HAT's hardware Audio ON/OFF slide switch (right edge, by the "Audio" label) must be ON.** - With it off, the codec still answers on I2C and every software check looks right, but all - recordings are exactly zero. That cost an hour of debugging; check it first. -- Mixer: `Left/Right Input Mixer Boost Switch` default to off and must be on. Settings in use: - Capture 47/47 (+18 dB), `Left/Right Input Boost Mixer LINPUT1/RINPUT1` volume 1, input mixer boost - on. Singing at arm's length: RMS -34 to -49 dBFS, peaks -18 dBFS, no clipping, channels within - 1-2 dB. Saved with `alsactl store` and exported as `pi/wm8960_asound.state`; restoring at boot is - not yet verified. No speakers on hand, so playback is untested. -- Not done: PWM (waits for servos). Ask before rebooting the Pi. +- **Display:** the ST7789 works on `lgpio` through Blinka (`rpi-lgpio`, not `RPi.GPIO`). Pass `cs=None` + (the kernel drives CE0; claiming GPIO8 raises "GPIO busy"). The 240x240 glass sits 80 rows into the + controller's RAM, `y_offset=80`, for every rotation (the driver rotates in software and leaves the + scan direction fixed). The mounted pose (microSD end up, USB-C edge on the left) needs the content + turned 90° clockwise: `rotation=270`. The face pushes raw RGB565 with `_block` at ~27-28 fps. +- **Renderer** (`render.py`, no ROS imports): a numpy port of `docs/face-prototype.html`, flat face only + (head and low-poly shapes are not ported). Phosphor and vector styles; tone map is a lookup table, + glow at half resolution, rotation a free `np.rot90`; parameters are clamped. States: `sleeping idle + listening thinking looking speaking`; `looking` returns to the state it interrupted. +- **Node** (`face_node.py`): subscribes `/hermes/state`, `/joint_states`, `/vision/faces`, + `/camera/camera_info`, `/audio/activity`, `/audio/scope/{mic,speaker}`; starts in `sleeping`; logs + every state change (`journalctl -u robot`), so behaviour can be checked without watching the panel. + `fps` can be changed live (the thermal guard does). +- **State arbitration** (`arbiter.py`): sleeping (agent side only) > speaking > listening > thinking > + idle. **Room sound never wakes a sleeping face.** +- **Agent heartbeat:** `/hermes/state` doubles as the heartbeat. After `agent_timeout_s` (10) without + it the face logs `agent gone, sleeping` and sleeps. +- **Gaze:** eyes glance toward the best face (a named one first, then the biggest), smoothed, held + 2.5 s, then drift back; sleeping ignores it. Image x flips on the display (a person on the image's + right is on the robot's right, the viewer's left). The renderer's pan sign was wrong (inherited from + the prototype) and is fixed: positive pan = the robot's left = the viewer's right; positive tilt = up, + as the MCP `turn_head` documents. Never seen with a moving head. +- **Voice trace:** while fresh scope data exists, the mouth draws the real waveform (speaker when + speaking, mic when listening), auto-scaled; the owner confirmed it follows how close and loud they are. +- Not done: head/low-poly shapes, DotStar LEDs, a button mute. Open: the IPS panel's black reads dark + gray on camera at full backlight; dimming needs a software PWM on GPIO26. -## Package 0b (USB link): Pi side and talos side working +## Camera and vision (`ros/hermes_vision`, `camera_ros`) -- Pi: `dtoverlay=dwc2,dr_mode=peripheral`; `robot-usb-gadget.service` builds a configfs gadget - (NCM `usb0` + ACM `ttyGS0`, host MAC `02:68:65:72:6d:01`, robot MAC `02:68:65:72:6d:02`) before - networking; `serial-getty@ttyGS0` is enabled; `/etc/netplan/60-usb0.yaml` gives `usb0` - `10.55.0.2/30` with no gateway, so the default route stays on Wi-Fi. Installed by - `pi/install-usb-gadget.sh` (a backup of `config.txt` is `config.txt.bak-usbgadget`). NCM works on - the Pi 4 with kernel 1065; ECM was not needed. -- **netplan gotcha:** match the gadget interface by `name: usb0`. A `macaddress:` match becomes - `PermanentMACAddress=` in networkd, which a gadget interface lacks, so it stays unmanaged. -- talos (Omarchy, NetworkManager): `host/omarchy/robot-usb-link.sh` (needs sudo) adds profile - `hermes-robot-usb`: `10.55.0.1/30` on the interface with MAC `02:68:65:72:6d:01`, priority 100, - never-default. Verified from talos: ping ~2.4 ms, SSH over the link. -- **Firewall:** `robot-firewall.service` loads `pi/nftables-robot.nft` before networking at every - boot (`pi/install-firewall.sh`; it must be loaded before the PipeWire network server is enabled). - It accepts 4713/8765/8766 only on `usb0` from `10.55.0.1`, drops them elsewhere, and drops DDS on - every interface but loopback. Verified: from talos, 4713 connects over `usb0` and times out over - Wi-Fi, while an unused port on the Pi's Wi-Fi address refuses instantly (so only 4713 is blocked). - SSH is deliberately left open on all interfaces (key-only), for V2; the owner's call. -- Not yet verified: serial login prompt (talos's `/dev/ttyACM0` is `root:uucp`; use - `sudo screen /dev/ttyACM0 115200`). The `hermes` account has no password, so the console shows a - login prompt but cannot log in until the owner sets one; no autologin by design. Also not done: - the Pi booting with Wi-Fi off, replug and second-host recovery, - `get_throttled` after a servo sweep on the splitter, the NixOS snippet, and straylight/other-host - testing. Currently the Pi is powered from talos's USB port (bench test, no servos). +- **Camera: only the Raspberry Pi libcamera fork works.** The OV5647 is detected (`ov5647 10-0036`), but + Ubuntu's libcamera 0.2.0 dies in the Pi IPA (`assertion "it != buffers_.end()" in prepareIsp()`) and + ROS Jazzy's 0.7.2 with `FATAL … A list of V4L2 controls requires a ControlInfoMap`, deterministically + for every format and size. The fork (commit `6c1dd9d`, built with `-Dpipelines=rpi/vc4 -Dipas=rpi/vc4 + -Dv4l2=false …` into `/opt/libcamera-rpi`) captures; `camera_ros` 0.7.0 is built from source against + it, and the fork must come first in `LD_LIBRARY_PATH` (`robot-env.sh` does it). Stream: 1296x972 at + 15 fps (`FrameDurationLimits` in `robot.yaml`); `/camera/image_raw`, `…/compressed`, `camera_info`. + `pi/capture_still.py` takes a still with the fork's `cam`. Frames are padded and B,G,R in memory. +- **Calibration done** (`pi/calibration/…_1296x972.yaml`, restored by the installer): 18 of 24 views, + reprojection error 0.296 px, focal length ~1153 px, centre (653, 463), small distortion with `k3` held + at zero. The first pass fit 0.41 px but with cancelling k1/k2/k3 because the board never reached the + edges; the tool now fixes `k3`, reports coverage, drops blurred outlier views and can re-solve saved + views (`--from-views`). **The top-left and bottom-right corners were never reached**, so the model is + least certain there. The result is valid for the current focus and image size; a changed focus + invalidates it. Tool: `python3 -m hermes_vision.calibrate --cols 9 --rows 6 --square-mm ` + (`docs/checkerboard-9x6.png` can be shown on a screen; the square size only scales distance, not the + intrinsics). +- **Detection** (`detect.py`, no ROS imports): OpenCV YuNet 2023mar (Haar fallback). It needs OpenCV + 4.8+ (apt has 4.6), so `~/venv` carries `opencv-python-headless==4.10.0.84` with `numpy<2` (the system + NumPy stays 1.26.4 for ROS), and the venv is first on `PYTHONPATH`; input sides must be multiples of 32. +- **Tracking** (`tracking.py`): faces are matched across frames by box overlap, so a person keeps one + track id; identity is re-resolved at most once a second. +- **Node** `face_detector` publishes `/vision/faces` (`vision_msgs/Detection2DArray`): boxes in + full-size camera pixels, `Detection2D.id` = the track id, `results[0]` the face, `results[1]` the + identity if recognised (class_id = name, score = cosine similarity, not a probability). An unknown + face has no `results[1]`: a name is never made up. `rate_hz` (4) can be changed live. +- **Recognition is opt-in** (`recognize.py`, `people.py`): SFace embeddings on YuNet's landmarks. Nothing + is stored until someone is enrolled with `python3 -m hermes_vision.people enroll NAME`; only + 128-number embeddings are kept, in `~/.local/share/hermes/faces.json` (mode 0600, dir 0700, written + atomically), never images. `list`, `forget`, `check`, `enroll-images` too. **The owner is enrolled + (10 samples) and was recognised in 29 of 31 detections live (similarity ~0.9).** A throwaway-database + test recognised 8 of 9 unenrolled frames of the same person, down to side profiles; an unrelated photo + scored 0.16. Not tested: a second person live. Names are never logged (a count is). -## Face (`ros/hermes_face`) +## Audio (`ros/hermes_audio`, `pi/overlays`, PipeWire) -- `render.py`: numpy port of `docs/face-prototype.html`, **flat face only** (head and lowpoly not - ported). Phosphor and vector styles. States: `sleeping idle listening thinking looking speaking`. - `sleeping` (new) is the "no agent connected" pose: eyes closed to thin lines, mouth short, dim and - slowly breathing. `looking` returns to the state it interrupted. Tone mapping is a lookup table - (RGB and RGB565 tables), glow is computed at half resolution, rotation is a free `np.rot90`. -- `display.py`: ST7789 driver plus an async pusher (render overlaps the SPI transfer). -- `face_node.py`: node `hermes_face`; subscribes `/hermes/state` (`std_msgs/String`) and - `/joint_states` (`pan_joint`, `tilt_joint`, radians); starts in `initial_state` (`sleeping`), so it - runs with nothing else up. Parameters: `rotation` (270), `x_offset` (0), `y_offset` (80), - `baudrate`, `fps`, `output` (`st7789` or `none`), `phosphor` (P7), `style`, `persistence_ms`, - `focus`, `jitter`, `intensity`, `initial_state`. -- `demo.py`: no ROS. `python -m hermes_face.demo --sequence` cycles the states on the panel; - `--png-dir DIR` writes a PNG per state without hardware. -- Tests: `python3 -m pytest test` in `ros/hermes_face` (25 pass on the Pi). Renderer parameters - (`focus`, `persistence_ms`, `intensity`, `jitter`) are clamped; `fps` is clamped to 1-60. -- Measured on the Pi 4: ~9 ms per frame in isolation. With the CPU at full clock: 24% of a core at - 20 fps and 40% at 30 fps (target was under 40% at 20 fps or more). Under `ondemand` the same - load reads 40% at 20 fps and ~67% at 30 fps. No C++ rewrite was needed. -- **Gaze:** the node subscribes to `/vision/faces` (and `/camera/camera_info` for the image size) - and the eyes glance toward the best face: a recognised one first, then the biggest. Image x flips - on the display (a person on the image's right is on the robot's right, the viewer's left); the - glance is smoothed, held 2.5 s, then drifts back to centre. Sleeping ignores it. The pan sign in - the renderer was wrong (carried over from the prototype) and is fixed: positive pan = the robot's - left = the viewer's right, positive tilt = looks up, as the MCP `turn_head` tool documents. - **Confirmed by the owner on the real display (2026-10-04):** the glance directions look right. - An earlier run showed nothing because the one-off wake message was lost (volatile QoS drops a - sample published before discovery matches); test publishers must republish, as `hermes_mcp` - will. The node logs every state change (`state: idle (agent=..., audio=...)`), so behaviour can be - checked from `journalctl -u robot-face` without watching the panel. -- **Agent heartbeat:** `/hermes/state` doubles as the agent's heartbeat. `hermes_mcp` must republish - its state about once a second; after `agent_timeout_s` (default 10, 0 disables) with none, the - face logs `agent gone, sleeping` and sleeps, so "sleeping" means "no agent connected". Verified: - woke on messages, slept 10 s after the last. -- Not yet: audio levels for the mouth (`/audio/levels`), `hermes_mcp` publishing states, launch - file in `hermes_bringup`, head/lowpoly shapes, DotStar LEDs, button mute. -- Open: IPS black level. The backlight is on at full brightness, so black reads dark gray on - camera. Dimming it needs a software PWM on GPIO26 (not a hardware PWM pin). +- **The stock `wm8960-soundcard` overlay fails on Ubuntu 24.04** (`wm8960: No MCLK configured`): it puts + the 12.288 MHz fixed clock on the card's codec link, but the codec driver reads `mclk` from its own + node. `pi/overlays/hermes-wm8960.dts` adds it there; `pi/install-audio-overlay.sh` compiles it, sets + `dtparam=audio=off` and `dtoverlay=hermes-wm8960`. Capture then opens at 8-48 kHz. Address the card + by name (`hw:CARD=wm8960soundcard,DEV=0`): its index changes. +- **The HAT's hardware Audio ON/OFF switch (right edge) must be ON.** With it off, the codec still + answers on I²C and every software check looks right, but all recordings are exactly zero. +- **Mixer:** `Left/Right Input Mixer Boost` default to off and must be on. `pi/wm8960_asound.state` holds + the settings (capture 47, boost 1). **WirePlumber resets the codec's capture gain to +30 dB** when it + takes the device, so the installer sets PipeWire's source volume to 63% (its scale is cubic: -12 dB, + ALSA `Capture` 46, about +17 dB), which WirePlumber remembers; survives reboots (checked). Quiet-room + floor at this gain: about -43 dBFS; singing at arm's length peaks around -18 dBFS. +- **Node** `audio_state` taps the default source and sink monitor with `parec` (16 kHz mono) and + publishes `/audio/levels`, `/audio/activity` (speaking | listening | quiet) and trigger-locked + waveform windows `/audio/scope/{mic,speaker}` (only while listening or speaking). Audio itself never + goes through ROS. `Activity`: 3 s start-up calibration of an adaptive noise floor, margin 7 dB, 1.2 s + hold so pauses between words do not flicker, steady noise is absorbed after 8 s, the mic is ignored + while speaking. **The flicker retune is unit-tested but not re-confirmed live.** +- Verified: the `speaking` path (a tone played to the sink, no speaker attached, shows on the monitor); + the mic path with the owner's voice. Playback to a real speaker, echo cancel (`pi/pipewire/ + 60-echo-cancel.conf.optional`) and the host's `robot_mic` naming wait for the speakers. -## Audio levels and the literal voice trace (`ros/hermes_audio`, face integration) +## The MCP bridge and Hermes (`ros/hermes_mcp`) -- `hermes_audio` node `audio_state` taps the default source and the default sink's monitor with - `parec` (16 kHz mono) and publishes `/audio/levels` ([mic_dbfs, speaker_dbfs]), `/audio/activity` - (`speaking` | `listening` | `quiet`) and trigger-locked waveform windows `/audio/scope/{mic,speaker}` - (256 samples). Audio itself never goes through ROS. `hermes_audio.levels.Activity` debounces and - keeps an adaptive noise floor: a 3 s start-up calibration, then it drops fast and rises slowly, and - anything 'listening' without a gap for 8 s is absorbed as ambient noise. The microphone is ignored - while speaking (it would hear the robot). -- The face arbitrates (`hermes_face.arbiter`): sleeping (agent side only) > speaking > listening > - thinking > idle. **Room sound never wakes a sleeping face**: sleeping means "no agent connected". - `looking` stays a timed event owned by the renderer. While fresh (< 0.35 s) scope data exists, the - mouth draws the real waveform (speaker when speaking, mic when listening), auto-scaled. -- **WirePlumber resets the codec's capture gain** to 100% (+30 dB) when it takes the device, - overriding `alsactl`'s saved level. Fix: `pactl set-source-volume @DEFAULT_SOURCE@ 63%` (PipeWire's - scale is cubic: 63% is -12 dB from the +30 dB maximum, so ALSA `Capture` lands at 46, about +17 - dB). WirePlumber remembers it; recheck after a reboot. The quiet-room floor at this gain is about - -43 dBFS (preamp and mic noise). -- **Confirmed by the owner:** the mouth draws their live voice and its size follows how close and - loud they are (proportional). The activity detector was retuned after the log showed the face - flickering between listening and idle about twice a second: margin 7 dB (quiet-room noise peaks - sit at about +3 dB over the floor), hold 1.2 s so pauses between words do not drop it. The - flicker fix is covered by unit tests but not yet re-confirmed live (the retest window was silent). -- Verified on the Pi: the `speaking` path (a tone played to the sink, with no speaker attached, shows - on the monitor at -13.5 dBFS, `speaking` for its length, mic ignored); levels and scope publish; - 49 face tests, 10 audio tests. **Not yet verified:** the audio node as a service (it was run as a - transient unit). - -## The MCP bridge (`ros/hermes_mcp`) - -The only way in or out of the ROS graph. Serves MCP (SDK 2.3, streamable HTTP) on -`http://10.55.0.2:8765/mcp`, the USB address only (retrying the bind if the link is not up yet), with -a bearer token. **Tool names are the stable contract with Hermes**: `look()`, `faces()`, `list_people()`, and `turn_head` / -`head_position` only when a head is fitted. With no servos those two are not registered, by design -(`head: false`; the node refuses `head: true` until `hermes_head` exists). `look()` waits for the next -camera frame taken after the call (never a stale one) and passes the camera's JPEG through (1296x972, -~155 KB); it also publishes `looking` on `/hermes/state` so the face squints and flashes. -While an authenticated client is connected (in flight, or heard within `connected_grace_s` = 30) it -republishes `idle` on `/hermes/state` once a second: the agent heartbeat that wakes the face. Verified -end to end from talos over the USB link with a real MCP client: connects, lists `look`, `look()` -returns a JPEG in 0.08 s, a wrong token is refused, the Wi-Fi path is dropped by the firewall, and -the face went `looking` -> `idle` on connect and `agent gone, sleeping` about 40 s after the client -left. The token is in `/etc/robot-mcp.env` (root:hermes, mode 640), generated by -`pi/install-robot-service.sh` and never printed; read it with `sudo cat`. The MCP SDK is in `~/venv` -(`pip install "mcp>=2"`). The old Pi Zero files (`pi/robot_mcp_server.py`, `pi/robot-mcp.service`, -`pi/requirements.txt`, the old `pi/install.sh`, and `tools/`) were deleted (git has them). **Not yet verified:** a real Hermes client, and -whether Hermes passes MCP image content to the model (HANDOFF package 6). - -**Hermes on talos is wired to it (2026-10-05, with the owner's authorisation).** Hermes Agent v0.21.5 is -installed on talos (`~/.hermes`; hosted main model `grok-4.6` via xai-oauth; vision is auxiliary and -runs locally, `qwen3-vl-8b-abliterated` on straylight; the `ornith-*` models are the local ones on -straylight/dixie). `~/.hermes/config.yaml` gained an `mcp_servers.robot` block (URL -`http://10.55.0.2:8765/mcp`, `Authorization: Bearer ${ROBOT_MCP_TOKEN}`) and `~/.hermes/.env` gained -`ROBOT_MCP_TOKEN`; backups are `config.yaml.bak-20261005T002659-robot-mcp` and -`.env.bak-20261005T002659-robot-mcp`. `hermes mcp list` shows it enabled and **`hermes mcp test -robot` passes: connected in 789 ms, 1 tool (`look`)**, which is the HANDOFF acceptance check. A fresh -`HERMES_HOME` hangs on first-run setup, so test against the real one. - -**A real agent turn works, entirely on local models (2026-10-05):** `hermes chat -q "Call the look tool -once ... what you see" --oneshot -Q --provider custom:llama-router -m ornith-1.5-35b-a3b --max-turns 4` -(straylight's router; no hosted provider, so the image stayed on the LAN). Hermes called -`mcp__robot__look` (0.07 s), the model described the live photo correctly (blurry close-up of a board -with blue LEDs, a red label, camouflage fabric), and the robot's face went `looking` -> `idle`. -**Hermes does not hand MCP image content to the main model.** It saves the JPEG to -`~/.hermes/cache/images/img_.jpg` on talos (157,147 bytes, the same as the robot sent) and -returns a 72-character reference; the model then calls Hermes's `vision_analyze`, which ran on the local -vision model (`qwen3-vl-8b-abliterated` on straylight) in 29 s. That is HANDOFF package 6's fallback and -Hermes does it itself, so `look()` needs no change. Two consequences: each look() leaves a photo of the -room in talos's Hermes image cache (Hermes's retention, not the robot's), and a look-and-describe turn -takes ~70 s on the local models, mostly the vision call. - -### Hermes's feedback on the bridge (2026-10-05) and what was done - -Hermes (on talos, from `~/.hermes/cache/scratch/hermes-bot-v0-face-identity.md`) asked for recognitions -instead of raw photos. **Built:** `faces()` returns the faces in view as structured JSON with no photo -(`track_id`, pixel and normalised `bbox`, detection `confidence`, `seen_for_s`, and `identity` -`{name, similarity}` or `null`); `look(send_image="always"|"never")` now returns that same JSON as a -text block plus the photo (default kept as before: the image, so scene description still works); -`list_people()` gives names and sample counts; a `robot://status` resource (hardware present or -absent, what is detected, enrolled count, tools) and prompts `who_is_here` and `look_around` fill the -previously empty `list_resources` / `list_prompts`; `/vision/faces` carries a persistent track id -(`hermes_vision.tracking.FaceTracker`, matching by box overlap, identity re-checked once a second; -`Detection2D.id` is the track id, `results[0]` the face, `results[1]` the identity if recognised). -Verified with a real Hermes turn on the local `ornith` model: it called `faces()` (0.03 s, 593 -characters, no image, no vision call) and answered that Kyle was there at 0.78 similarity. -**Declined, deliberately:** embeddings in responses (biometric templates, meaningless to an LLM, -and the registry stays on the robot); age/gender/hat attributes (no model for them here, and guessing -age and gender from faces is unreliable and sensitive); auto-updating a person's face data from new -captures (it lets a stranger slowly become "Kyle" with nobody deciding that); a remote `enroll` tool -(a prompt could make the robot call anyone "Kyle": enrolment stays the owner's step on the robot, -`python3 -m hermes_vision.people enroll NAME`; a gated enrol tool, off by default, is possible if the -owner wants it); detecting people, objects or motion (only faces exist, and `robot://status` says so). -`look()`'s default was not flipped to JSON-only, because the image is what lets Hermes describe a -scene; that default is the owner's call. - -### Hermes bug report: "faces and list_people are not registered" (2026-10-05) — not a bridge bug - -Checked, not assumed: a raw MCP client lists `faces`, `look`, `list_people`; `hermes mcp test robot` -(fresh connection) discovers all three; there is no tool filter in `~/.hermes/config.yaml`. Hermes's own -`agent.log` explains the report: my robot restart at 01:17 ended the MCP session (`keepalive failed ... -Session not found`), Hermes reconnected and logged `registered 7 tool(s): mcp__robot__faces, look, -list_people, ...` at 01:18:46, and a session that started afterwards (`...011845...`) called -`mcp__robot__faces` fine. The reporting session (`...003532...`) started at 00:35, before `faces` -existed, and keeps its start-time snapshot of the tool table: it still reaches live things -(`list_prompts`, `read_resource`) but gets `'mcp__robot__faces' is not a known tool name`. Fix on the -owner's side: start a new Hermes session after the robot's tool set changes. For Hermes: refresh a -session's tool table (and its deferred-tool index) when the MCP server is re-registered. We keep the -bridge's stateful sessions on purpose: the `Session not found` after a restart is what makes Hermes -reconnect and re-register (stateless HTTP would hide a changed tool set from a connected client). - -## Face recognition (opt-in) - -`hermes_vision.recognize` / `people`: SFace embeddings on YuNet's landmarks. Nothing is stored until -someone is enrolled (`python -m hermes_vision.people enroll NAME`, live from the camera topic, or -`enroll-images`); only 128-number embeddings are kept, in `~/.local/share/hermes/faces.json` (mode -0600, directory 0700, written atomically), never images. `list`, `forget NAME` and `check IMAGE` -manage and test it. The detector node puts a recognised name in `Detection2D.id` and picks up -database changes without a restart; its log prints a count, not names. Checked with a throwaway -database: 8 of 9 unenrolled frames of the same person were recognised, down to side profiles -(similarity 0.52-0.66), the miss being a head tipped back (0.34 against the 0.363 threshold); an -unrelated photo scored 0.16. **Nobody is enrolled yet**; the owner enrols themselves live. - -## Next - -1. **Camera calibration: tool built and tested, not yet run.** `python3 -m hermes_vision.calibrate - --cols 9 --rows 6 --square-mm ` (after `source ~/robot-env.sh`, robot running) collects 20 - varied views of a checkerboard (`docs/checkerboard-9x6.png` can be shown on a screen; measure a - square), solves with OpenCV and writes the camera_info YAML camera_ros looks for - (`~/.ros/camera_info/ov5647__base_soc_i2c0mux_i2c_1_ov5647_36_1296x972.yaml`: the name includes the - resolution). The solver recovers a known synthetic camera to within 1%. It needs the owner holding a - board in front of the camera, and the lens focus set first (the lens turns if not glued; earlier - close-range frames were blurry). V2's perception needs the result. -2. Waiting on hardware: the servos (see "Arrival day" above), the speakers - (playback, echo cancel, the `robot_mic` naming on the host, Hermes talking aloud), the fan (before - any enclosure), the splitter, the printed enclosure (measure first, see `enclosure/README.md`). -3. Still open: replug and second-host recovery of the USB link, the serial console login, an - `agent_timeout_s` check against a long idle Hermes session, the retuned listening detector live - (unit-tested only), a second person in frame, running `pi/install.sh` on a blank card, and the - NixOS host config (skipped for now by the owner). -4. A new Omarchy host with a GPU (it runs the local `ornith` model) is arriving: it needs only - `host/omarchy/robot-usb-link.sh` and its SSH key added to the Pi. +- Serves MCP (SDK 2.3, streamable HTTP) on `http://10.55.0.2:8765/mcp`: the USB address only (it retries + the bind until the link is up), bearer token in `/etc/robot-mcp.env` (root:hermes, mode 640, made by + the installer, never printed; read it with `sudo cat`). **Tool names are the stable contract with + Hermes.** Tools: `look(send_image="always"|"never")` (the faces summary as JSON text plus the next + fresh JPEG, never a stale frame; it also makes the face squint), `faces()` (the faces in view, no + photo: `track_id`, `bbox`, `confidence`, `seen_for_s`, `identity` `{name, similarity}` or `null`), + `list_people()` (names and sample counts only), and `turn_head` / `head_position` **only when a head + is fitted** (`head: false` now; the node refuses `head: true` until `hermes_head` exists). Resource + `robot://status` (hardware present or absent, what is detected, enrolled count, tools, and health: + temperature, thermal level, throttle flags now and since boot) and prompts `who_is_here`, + `look_around` make it discoverable. Sessions are stateful on purpose (see the Hermes note below). +- **Heartbeat:** while an authenticated client is connected (in flight, or heard within + `connected_grace_s` = 30) it republishes `idle` on `/hermes/state` once a second; the face wakes on + connect and sleeps about 40 s after the client leaves. +- **Hermes on talos is wired to it** (with the owner's authorisation). Hermes Agent v0.21.5 is in + `~/.hermes` (hosted main model `grok-4.6` via xai-oauth; vision is auxiliary and local, + `qwen3-vl-8b-abliterated` on straylight; the `ornith-*` models are the local ones on straylight and + dixie). `config.yaml` has an `mcp_servers.robot` block and `.env` has `ROBOT_MCP_TOKEN`; backups: + `config.yaml.bak-20261005T002659-robot-mcp`, `.env.bak-20261005T002659-robot-mcp`. `hermes mcp test + robot` passes. A fresh `HERMES_HOME` hangs on first-run setup, so test against the real one. +- **Real agent turns, entirely on local models:** `hermes chat -q "..." --oneshot -Q --provider + custom:llama-router -m ornith-1.5-35b-a3b --max-turns 5`. `look()` works; **Hermes does not hand MCP + images to the main model**: it saves the JPEG to `~/.hermes/cache/images/` on talos and returns a short + reference, and the model calls `vision_analyze` (local vision model, ~29 s). So a look-and-describe turn + takes ~70 s, and each look leaves a photo in talos's Hermes cache (Hermes's retention). `faces()` is + the cheap path: 0.03 s, no image, and Hermes answered "Kyle is at the desk, similarity 0.78". +- **Hermes's feedback (`~/.hermes/cache/scratch/hermes-bot-v0-face-identity.md`).** Built: `faces()`, + `look` returning that JSON, `list_people`, `robot://status` and prompts (previously empty), track ids. + **Declined, deliberately:** embeddings in responses (biometric templates, meaningless to an LLM; the + registry stays on the robot); age/gender/hat (no model, and unreliable and sensitive); auto-updating + face data from new captures (lets a stranger slowly become "Kyle"); a remote `enroll` tool (a prompt + could make the robot call anyone "Kyle": enrolment stays the owner's step; a gated tool, off by + default, is possible if wanted); detecting people, objects or motion (only faces exist). `look()`'s + default stays photo-plus-faces, because the photo is what lets Hermes describe a scene: **flipping it + to JSON-only is the owner's call.** +- **Hermes bug report "faces and list_people are not registered" was not a bridge bug.** A raw client and a + fresh `hermes mcp test` list all three; Hermes's own `agent.log` shows it registering all seven tools + at 01:18:46 after a robot restart, and a later session calling `faces()` fine. The reporting session + started at 00:35, before `faces` existed, and keeps a start-time snapshot of its tool table (live calls + like `list_prompts` still work, which matches the report). Fix: start a new Hermes session after the + robot's tool set changes; Hermes should refresh a session's tool table on re-registration. The bridge + keeps stateful sessions on purpose: the `Session not found` after a restart is what makes Hermes + reconnect and re-register. ## Thermal guard (`ros/hermes_thermal`) -Built because the Pi has no fan (about 60 C open-air with everything running) and an enclosure will be -hotter. `thermal_guard` reads the CPU temperature (`/sys/class/thermal`) and `vcgencmd get_throttled`, -publishes `/health/temperature`, `/health/thermal_level` (ok | warm | hot | critical) and -`/health/throttled`, and sheds load through the target nodes' own live parameters: the camera's -`FrameDurationLimits`, the face's `fps` and the detector's `rate_hz` (the face and detector gained live -parameter callbacks for this). Levels enter at 72 / 77 / 82 C (the Pi throttles at about 80); heating is -acted on at once, cooling goes one level at a time, only below the threshold minus 3 C and after a 20 s -dwell. Rates scale by 1.0 / 0.7 / 0.45 / 0.2 with floors (camera 2 fps, face 8, detector 1 Hz). The -"normal" rates are parameters of the guard and a bringup test asserts they equal the real ones in -`robot.yaml`, so restoring is exact and survives the guard restarting. `fake_temp_c` forces a -temperature for testing. `robot://status` now carries a `health` block (temperature, level, throttle -flags now and since boot, which doubles as a power-supply check). -**Verified live on the robot** by forcing 73 / 78 / 85 C and back: the camera, face and detector rates -followed (measured camera 16.5 -> 11.5 -> 7.2 -> 3.5 Hz and back to 16.5), cooling stepped down critical --> hot -> warm -> ok exactly one dwell apart, and the guard stayed the same process throughout. -**A live bug the unit tests missed:** one logging call that warned when heating and informed when -cooling raised `ValueError: Logger severity cannot be changed between calls` in rclpy, killing the -guard on its first cooling step (the launch file respawned it, which only looked like a recovery). Fixed -with separate call sites, the tick now catches and logs any error instead of dying, and a test drives -the real node through a full heat-then-cool run; a mutation check (the old logging in a throwaway -copy) confirmed that test fails on exactly that error. Not yet tested under a real thermal load. -When the servos arrive, watch the under-voltage flag in `robot://status` during a sweep. +The Pi has no fan. `thermal_guard` reads the CPU temperature and `vcgencmd get_throttled`, publishes +`/health/temperature`, `/health/thermal_level` (ok | warm | hot | critical) and `/health/throttled`, and +sheds load through the target nodes' live parameters: the camera's `FrameDurationLimits`, the face's `fps`, +the detector's `rate_hz`. Levels enter at 72 / 77 / 82 °C (the Pi throttles at ~80); heating is acted on +at once, cooling goes one level at a time only below the threshold minus 3 °C and after a 20 s dwell. +Rates scale by 1.0 / 0.7 / 0.45 / 0.2 with floors (camera 2 fps, face 8, detector 1 Hz). Its "normal" +rates are its own parameters, and a bringup test asserts they equal the real ones in `robot.yaml`, so +restoring is exact and survives a restart. `fake_temp_c` (`ros2 param set /hermes_thermal fake_temp_c +85.0`, -1 to stop) forces a temperature. **Verified live** by forcing 73 / 78 / 85 °C and back: measured +camera rate 16.5 -> 11.5 -> 7.2 -> 3.5 Hz and back, cooling stepped down one dwell apart, same process +throughout. **A live bug the unit tests missed:** one log line that warned when heating and informed when +cooling raised rclpy's `Logger severity cannot be changed between calls` and killed the guard on its +first cooling step (the respawn hid it). Fixed (separate call sites), the tick now survives any error, and +a test drives the real node through heat-then-cool; a mutation check confirmed it fails on the original +bug. Not yet tested under a real thermal load. + +## USB link (HANDOFF package 0b) + +- Pi: `dtoverlay=dwc2,dr_mode=peripheral`; `robot-usb-gadget.service` builds a configfs gadget (NCM `usb0` + + ACM `ttyGS0`; host MAC `02:68:65:72:6d:01`, robot MAC `…:02`) before networking; + `/etc/netplan/60-usb0.yaml` gives `usb0` `10.55.0.2/30` with no gateway (the default route stays on + Wi-Fi). NCM works on the Pi 4 (ECM was not needed). **netplan must match `usb0` by name, not MAC:** a + `macaddress:` match becomes `PermanentMACAddress=`, which a gadget interface lacks, so it stays unmanaged. +- Host (Omarchy, NetworkManager): `sudo host/omarchy/robot-usb-link.sh` adds profile `hermes-robot-usb` + (`10.55.0.1/30`, matched by the MAC, never-default). Ping ~2 ms. +- **Wi-Fi-off acceptance passed:** with Wi-Fi blocked (`rfkill block wifi`, remembered across reboots), + live and after a reboot, the Pi came up on the USB link alone and everything worked. With no internet + there is no NTP, so the clock is unsynchronised (roughly right). +- Not yet verified: replug and second-host recovery, and the serial console login (talos's `/dev/ttyACM0` + is `root:uucp`; `hermes` has no password, so the console shows a prompt but cannot log in; no autologin + by design). + +## Owner's decisions + +ROS never leaves the Pi (`hermes_mcp` is the only bridge); SSH stays open for V2; NixOS host config is +skipped for now; no remote `enroll`; `look()`'s default is open; the face may take up to a minute to +appear at boot; everything should be ROS. A new Omarchy host with a GPU (it runs the local `ornith` +model) is arriving: it needs only `host/omarchy/robot-usb-link.sh` and its SSH key on the Pi. + +## Gotchas + +- `ros2 node list` with its default short window shows only some nodes: use `--no-daemon --spin-time 6`. +- A one-off message published before ROS discovery matches is lost (volatile QoS): republish, as + `hermes_mcp` does every second. Anything that sets `PYTHONPATH` by hand must prepend, not replace. +- A shell that sourced `robot-env.sh` before a build cannot import the newly built package: re-source. +- rclpy needs each logging call site to keep one severity. ## Arrival day: the neck (servos), planned and deliberately not built ahead of the hardware -The servos are expected Thursday or Friday at the earliest. The real unknowns cannot be faked (which -`pwmchip` the overlay creates, its permissions on Ubuntu, each servo's real pulse range, whether the -servos brown out the Pi), so `hermes_head` waits for the hardware. The order, with what to verify at -each step: +Expected Thursday or Friday at the earliest. What matters cannot be faked (which `pwmchip` the overlay +creates, its permissions on Ubuntu, each servo's real pulse range, whether they brown out the Pi), so +`hermes_head` waits. Order, with what to verify: 1. **Wire it:** pan servo to the HAT's GPIO12 port, tilt to GPIO13; the "3pin Volt" jumper is already - verified at 5 V. Power matters: servo spikes on a host USB port may brown the Pi out, so use the - 5.1 V 3 A supply through the splitter if it has arrived. Horns off for the first moves. + verified at 5 V. Servo spikes on a host USB port may brown the Pi out: use the 5.1 V 3 A supply + through the splitter if it has arrived. Horns off for the first moves. 2. **Overlay (needs a reboot, ask first):** add `dtoverlay=pwm-2chan,pin=12,func=4,pin2=13,func2=4` under - `[all]` in `/boot/firmware/config.txt` (`dtparam=audio=off` is already set; the Pi's headphone audio - shares PWM hardware). Add it to `pi/install-*.sh`, not by hand. -3. **sysfs:** find the chip (`/sys/class/pwm/pwmchip*`; it was not assumed to be 0), export channels 0 - and 1, `period=20000000`. Add a udev rule that gives a group (not root) write access, as Ubuntu's - defaults differ from Raspberry Pi OS. -4. **Measure each servo:** centre (1.5 ms), then find the safe min and max pulse with the horn off; set - `ROBOT_SERVO_MIN_US` / `MAX_US` per servo (start 500 / 2400), the pan and tilt trims and invert flags. -5. **Write `hermes_head`:** `PwmHead` on sysfs, a ~0.3 s interpolated move, PWM off after ~1 s idle (stops - hold-jitter), limits (pan 80, tilt 45), `/joint_states` for `pan_joint` / `tilt_joint` in radians. - Convention (the MCP contract): positive pan = the robot's left, positive tilt = up. Unit-test the - logic against a fake sysfs tree, then verify on the Pi: `vcgencmd get_throttled` must stay `0x0` - through a full sweep while the face is running. -6. **Wire it in:** the bridge refuses `head: true` today. Give it a head backend that calls - `hermes_head`, set `head: true` in `robot.yaml`, and check `turn_head` / `head_position` appear and - that a new Hermes session sees them. Check the face's eyes follow the real joints the right way - (the sign was fixed in software, never seen with a moving head). -7. **URDF:** `hermes_description` from the enclosure STLs' `pose_*` parts, frames `base_link`, - `pan_link`, `tilt_link`, `camera_link`, `camera_optical_frame`, so `/tf` follows the head. -8. **Mounted camera:** set the flip (`orientation` parameter) for how it is actually mounted, re-check - the focus (a changed focus invalidates `pi/calibration/`), and set `ROBOT_FACE_ROTATION` if the - display ends up mounted differently from the portrait pose. + `[all]` in `config.txt` (`dtparam=audio=off` is already set: the Pi's headphone audio shares PWM + hardware). Add it to `pi/install-*.sh`, not by hand. +3. **sysfs:** find the chip (`/sys/class/pwm/pwmchip*`; do not assume 0), export channels 0 and 1, + `period=20000000`. A udev rule should give a group, not root, write access. +4. **Measure each servo:** centre (1.5 ms), then the safe min and max pulse with the horn off; set + `ROBOT_SERVO_MIN_US` / `MAX_US` per servo (start 500 / 2400), the trims and invert flags. +5. **Write `hermes_head`:** `PwmHead` on sysfs, a ~0.3 s interpolated move, PWM off after ~1 s idle, + limits (pan 80, tilt 45), `/joint_states` for `pan_joint` / `tilt_joint` in radians (positive pan = + the robot's left, positive tilt = up). Unit-test against a fake sysfs tree, then verify on the Pi: + `vcgencmd get_throttled` stays `0x0` through a full sweep with the face running; watch the + under-voltage flag in `robot://status`. +6. **Wire it in:** give the bridge a head backend that calls `hermes_head`, set `head: true`, check + `turn_head` / `head_position` appear to a new Hermes session, and that the face's eyes follow the real + joints the right way. +7. **URDF:** `hermes_description` from the enclosure STLs' `pose_*` parts, frames `base_link`, `pan_link`, + `tilt_link`, `camera_link`, `camera_optical_frame`. +8. **Mounted camera:** set the flip (`orientation`) for how it is mounted, re-check the focus (a changed + focus invalidates `pi/calibration/`), and the display rotation if it is mounted differently. + +## Open items + +1. Waiting on hardware: servos (above), speakers (playback, echo cancel, Hermes talking aloud, the host's + `robot_mic` naming), the fan, the splitter, the printed enclosure (measure first: `enclosure/README.md`). +2. Unconfirmed: the retuned listening detector live; a second person in frame; replug and second-host + recovery; the serial console; `agent_timeout_s` over a long idle Hermes session; the calibration's + uncovered corners (retake with the board in the top-left and bottom-right); `pi/install.sh` on a + blank card (back up the working SD card first: it holds all of this). +3. Optional: a gated `enroll` tool, if the owner wants one. ## Repository Branch `master`. `.gitignore` covers `.remember/`, caches, `.venv/`, `*.orig`, `*.rej` and -`pi/50-cloud-init.yaml`. Layout: `ros/` (five packages), `pi/` (everything installed on the Pi), -`host/` (the owner's machines: `host/omarchy/` has the USB link script, Hermes and PipeWire config and -the launcher; `host/check-pi-platform.sh`), `enclosure/`, `docs/face-prototype.html`. `README.md` is -rewritten for this build. **`pi/install.sh` is the one installer** (re-runnable: run twice on the live -robot, the second run changed nothing, exit 0): it installs ROS, builds the libcamera fork (pinned -commit) and `camera_ros`, sets up the venv, models, workspace, USB gadget, audio overlay, firewall, -PipeWire, mixer and services. It has not been run on a blank card (no spare card). HANDOFF.md is the -original brief and has drifted; where they differ, this file wins. +`pi/50-cloud-init.yaml`. `ros/` has `hermes_face`, `hermes_vision`, `hermes_audio`, `hermes_mcp`, +`hermes_thermal` and `hermes_bringup`; `pi/` is everything installed on the Pi; `host/` is the owner's +machines (`host/omarchy/`: the USB link script, Hermes and PipeWire config, the launcher; +`host/check-pi-platform.sh`); `enclosure/`; `docs/` (`face-prototype.html`, `checkerboard-9x6.png`). +Run a package's tests on the Pi with `python3 -m pytest test` in its directory after `source +~/robot-env.sh` (the face suite takes about a minute). ## Security notes (all resolved) -The automated review of the initial commit flagged three items, all from the Pi Zero design: -`ROBOT_MCP_BIND` defaulting to `0.0.0.0` (now the USB address only, with a bearer token), PipeWire on -TCP 4713 on every interface (now protected by the boot-time firewall, which loads first), and the -single-IP any-interface nftables file (rewritten: usb0 from 10.55.0.1 only). Later reviews added: -DDS is blocked on every interface and discovery is local-only; recognised names are not logged; node -parameters are clamped. SSH stays open on all interfaces (key-only) by the owner's choice, for V2. +The automated review of the initial commit flagged `ROBOT_MCP_BIND` defaulting to `0.0.0.0` (now the USB +address only, with a bearer token), PipeWire on TCP 4713 on every interface (now behind the boot-time +firewall, which loads first), and a single-IP any-interface nftables file (rewritten). Later reviews led +to: DDS blocked on every interface and local-only discovery; recognised names never logged; node +parameters clamped. The token and the owner's Wi-Fi password never go through the repo or logs (one +`netplan get` once printed the Wi-Fi password into a session; it was flagged to the owner). diff --git a/HANDOFF.md b/HANDOFF.md index 4222b1c..3987b73 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,5 +1,22 @@ # Hermes robot: Pi 4 + BrainCraft build — implementation handoff +> **Status note (2026-10-05): this is the original brief, kept as the record of intent. It has drifted.** +> Read `CONTEXT.md` for where things really stand. The main differences: +> - **ROS never leaves the Pi.** There is no `host/ros-shell` talking to the robot over DDS; `hermes_mcp` is +> the only bridge, and the firewall blocks DDS on every interface (section 3, package 0b, package 5). +> - **Stock libcamera does not work** on this kernel; the camera needs the Raspberry Pi libcamera fork, +> with `camera_ros` built against it (package 0a, package 3). The WM8960 overlay needed a fix too +> (package 1), and the HAT's Audio switch must be ON. +> - **MCP tools** are `look`, `faces`, `list_people` (and `turn_head` / `head_position` only when a head is +> fitted), not just `look` / `turn_head`. Hermes does not pass MCP images to the model; it saves the +> photo and the model calls its own vision tool, which is package 6's fallback, handled by Hermes. +> - **One launch file and one unit** run the whole robot; `pi/install.sh` is the single installer; the old +> Pi Zero files in section 4 are deleted; `omarchy/` became `host/omarchy/`. +> - Done since: the face (flat shape only), recognition (opt-in), the thermal guard, camera calibration. +> - Not built yet because the hardware has not arrived: `hermes_head` and the servos, speakers and echo +> cancel, the fan, the enclosure. The NixOS host config is skipped for now. + + This document is the brief for finishing the robot. It is written to be worked through by Claude Code with the owner at hand for the physical steps. Read it in full before changing anything; then work the packages in order, since later ones depend on earlier ones. diff --git a/README.md b/README.md index 39c4e08..0252148 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ through MCP. connected), idle, listening, thinking, looking, speaking. The mouth draws the actual voice, and the eyes glance toward whoever the camera sees. - **Ears and voice:** the HAT's two microphones and speakers, shared with your computer as ordinary - audio devices. + audio devices (the speakers are not fitted yet; the microphones work). - **Neck:** pan and tilt servos (`turn_head`), once they are fitted. - **One cable:** USB-C carries power and a private network link to your computer. Wi-Fi is a fallback. @@ -21,8 +21,8 @@ only speaks MCP. | Works and is tested | Waiting on hardware | |---|---| | Face on the display, always on at power-up, with a sleeping pose | Servos: `turn_head`, `head_position`, the pan/tilt URDF | -| Camera, face detection, opt-in recognition of people you enrol | Speakers: playback, echo cancel, Hermes talking out loud | -| Eyes follow faces; mouth draws your live voice | Fan (the Pi runs at about 62 °C open-air without one) | +| Camera (calibrated), face detection, opt-in recognition of people you enrol | Speakers: playback, echo cancel, Hermes talking out loud | +| Eyes follow faces; mouth draws your live voice | Fan (the Pi runs at about 60 °C open-air without one) | | MCP bridge: `hermes mcp test robot` passes, real Hermes turns call `look()` and `faces()` | USB-C splitter (the Pi is powered from your computer's port for now) | | Thermal guard: slows the camera, face and detector before the Pi throttles | | | USB network link, firewall, boots and works with Wi-Fi off | Printed enclosure (`enclosure/`) | @@ -71,8 +71,8 @@ door. 1. **Flash** Ubuntu Server 24.04 for the Pi (arm64, preinstalled image) and check its SHA256. 2. **Configure first boot** on the card's `system-boot` partition. In `user-data`, create the user `hermes` with key-only login and the hostname `hermes-bot`. For Wi-Fi, put the access point in - `network-config` (a template is `pi/50-cloud-init.yaml`; it holds your password, so it is - gitignored). cloud-init applies these on the *first* boot only: to change Wi-Fi later, edit + `network-config` (copy `pi/network-config.example` and fill it in; keep a filled copy out of the repo: `pi/50-cloud-init.yaml` + is gitignored for that). cloud-init applies these on the *first* boot only: to change Wi-Fi later, edit `/etc/netplan/50-cloud-init.yaml` on the Pi. 3. **Boot, then copy this repo to the Pi and run the installer:** @@ -90,9 +90,10 @@ door. 1. **Network link:** `sudo host/omarchy/robot-usb-link.sh` gives your end of the link `10.55.0.1/30`, matched by the gadget's fixed MAC, so any port and any machine works. The robot is `10.55.0.2`. -2. **Hermes:** the token is in `/etc/robot-mcp.env` on the Pi. Put it in `~/.hermes/.env` as - `ROBOT_MCP_TOKEN=`, and merge `host/omarchy/hermes-config-snippet.yaml` into - `~/.hermes/config.yaml`. +2. **Hermes:** the token is in `/etc/robot-mcp.env` on the Pi (`ssh hermes@10.55.0.2 sudo cat + /etc/robot-mcp.env`). Put it in `~/.hermes/.env` as `ROBOT_MCP_TOKEN=`, and merge + `host/omarchy/hermes-config-snippet.yaml` into `~/.hermes/config.yaml`. Start a new Hermes session + afterwards: a session keeps the tool list it started with. 3. **Audio (optional, needs the speakers):** `host/omarchy/50-robot-tunnel.conf` makes the robot's microphone and speaker appear as devices; `host/omarchy/hermes-robot` starts Hermes with them as the defaults. @@ -137,27 +138,37 @@ give MCP images to the main model: it saves the photo to its image cache and the restart robot`. - **Enrol yourself** so the eyes prefer you (numbers only are stored, never photos, in `~/.local/share/hermes/faces.json`): on the Pi, with the robot running and after `source - ~/robot-env.sh`, `python3 -m hermes_vision.people enroll YOURNAME`. `list` shows who is enrolled, `forget YOURNAME` removes you. + ~/robot-env.sh`, `python3 -m hermes_vision.people enroll YOURNAME`. `list` shows who is enrolled; + `forget YOURNAME` removes you. Enrolling is deliberately not an MCP tool: a prompt must not be able to + teach the robot a face. - **Update the code:** `rsync -a --delete --exclude __pycache__ ros/ hermes@:~/ros_ws/src/`, then `colcon build --symlink-install` in `~/ros_ws`, then `sudo systemctl restart robot`. - **Run a node by hand:** `source ~/robot-env.sh` first: it puts the libcamera fork and the right OpenCV on the path, and keeps ROS local. +- **Run the tests:** on the Pi, after `source ~/robot-env.sh`, `python3 -m pytest test` in each package + under `~/ros_ws/src/` (the face suite takes about a minute). A shell that sourced the environment + before a package was built cannot import it: source again. ## Calibrating the camera -Needed once, and again if you change the lens focus or the image size. Set the focus first (the lens -turns if it is not glued: focus at the distance you will use), then show `docs/checkerboard-9x6.png` -on a screen (or print it) and measure one square with a ruler. With the robot running, on the Pi, after -`source ~/robot-env.sh`: +This camera's calibration ships in `pi/calibration/` and the installer restores it, so a fresh install is +already calibrated. Redo it if you change the lens focus, the image size or the camera: a changed focus +invalidates it. Set the focus first (the lens turns if it is not glued: focus at the distance you will +use), then show `docs/checkerboard-9x6.png` on a screen (or print it) and measure one square with a +ruler. With the robot running, on the Pi, after `source ~/robot-env.sh`: ``` python3 -m hermes_vision.calibrate --cols 9 --rows 6 --square-mm ``` -Hold the camera 25 to 60 cm from the board and vary distance, tilt and position, keeping all of it in -view. It collects 20 different views, solves with OpenCV and writes -`~/.ros/camera_info/ov5647__base_soc_i2c0mux_i2c_1_ov5647_36_1296x972.yaml`. A reprojection error under -about 0.5 px is good. `sudo systemctl restart robot` to load it. +Hold the camera 25 to 60 cm from the board and vary distance, tilt and position, **including the edges +and corners of the camera's view** (the tool says which regions are still uncovered), and hold still: +blurred views are dropped as outliers. It collects 24 different views, solves with OpenCV (holding the +`k3` distortion term at zero, which a lens this mild does not need) and writes +`~/.ros/camera_info/ov5647__base_soc_i2c0mux_i2c_1_ov5647_36_1296x972.yaml`, plus the collected views so +`--from-views FILE` can re-solve them without recapturing. A reprojection error under about 0.5 px is good. +The square size only scales distances, not the lens model, so an approximate measurement is fine. +`sudo systemctl restart robot` to load it, and copy the YAML into `pi/calibration/` to keep it. ## Tuning and troubleshooting @@ -205,5 +216,5 @@ about 0.5 px is good. `sudo systemctl restart robot` to load it. | `pi/` | Everything installed on the Pi: `install.sh` and its parts, services, firewall, overlays. | | `host/` | Your computer's side: the USB link script, Hermes and PipeWire config, a platform check. | | `enclosure/` | OpenSCAD model, STLs and the print guide. Built from estimates: measure before printing. | -| `docs/face-prototype.html` | The browser prototype the face was ported from. | +| `docs/` | `face-prototype.html` (the browser prototype the face was ported from) and `checkerboard-9x6.png` (for calibration). | | `HANDOFF.md` | The original implementation brief. `CONTEXT.md` tracks where things really stand. | diff --git a/enclosure/README.md b/enclosure/README.md index 6bd8eb0..5de1fe7 100644 --- a/enclosure/README.md +++ b/enclosure/README.md @@ -73,8 +73,9 @@ accuracy. 1. **Base:** set the pan servo on its posts, shaft up, and screw it through the tabs. Press the bearing into the seat. Feed the USB-C cable in through the slot at the back of the base, up through the bearing. -2. **Turntable:** screw a double-arm horn into the pocket under the plate. Centre the servo (run the - software's `turn_head(0, 0)`, or use a servo tester), press the plate's hub into the bearing so the +2. **Turntable:** screw a double-arm horn into the pocket under the plate. Centre the servo (with a servo + tester, or a 1.5 ms pulse on its PWM channel as in "Arrival day" in `CONTEXT.md`; `turn_head(0, 0)` + works only once the head node exists and the bridge has `head: true`), press the plate's hub into the bearing so the horn meets the spline, and fit the horn's centre screw through the hole in the plate. 3. **Body onto the plate:** pass the power cable and the pan servo's lead up through the floor hole. Then screw the plate on from inside the body with 4 × M3, before the speaker goes in. diff --git a/host/omarchy/hermes-config-snippet.yaml b/host/omarchy/hermes-config-snippet.yaml index ae4e0ad..3d0e9b9 100644 --- a/host/omarchy/hermes-config-snippet.yaml +++ b/host/omarchy/hermes-config-snippet.yaml @@ -4,7 +4,10 @@ # It lives only on the Pi, in /etc/robot-mcp.env (read it with: ssh hermes@10.55.0.2 sudo cat /etc/robot-mcp.env). # The robot's firewall accepts this port only over the USB link, from 10.55.0.1. # While Hermes holds a connection the robot's face is awake; when it stops, the face falls asleep after -# about 40 seconds. A robot with no head fitted offers only look(); turn_head appears when servos do. +# about 40 seconds. Tools: faces() (who is in view: names, positions, tracking ids; instant, no photo), +# look() (the same plus a photo), list_people(); turn_head and head_position appear only once servos are +# fitted. A Hermes session snapshots the tool list when it starts: after the robot's tools change, start +# a new session. Resource robot://status says what hardware this robot has. mcp_servers: robot: diff --git a/pi/network-config.example b/pi/network-config.example new file mode 100644 index 0000000..27645b4 --- /dev/null +++ b/pi/network-config.example @@ -0,0 +1,24 @@ +# Example first-boot network config for the Pi, with Wi-Fi. COPY it to `network-config` on the card's +# `system-boot` partition and fill in SSID and PASSWORD (or keep a filled copy outside the repo: +# `pi/50-cloud-init.yaml` is gitignored for that, because it holds your password). +# +# cloud-init applies this on the FIRST boot only. To change Wi-Fi later, edit +# /etc/netplan/50-cloud-init.yaml on the Pi (mode 600, owner root), or bump `instance_id` in `meta-data` +# on the boot partition to force a fresh first boot. +# Spaces only, no tabs. Quote the SSID and password. +network: + version: 2 + + ethernets: + eth0: + dhcp4: true + optional: true + + wifis: + wlan0: + dhcp4: true + optional: false + regulatory-domain: US # set your country: the Pi may refuse some 5 GHz channels without it + access-points: + "SSID": + password: "PASSWORD" diff --git a/pi/pipewire/60-echo-cancel.conf.optional b/pi/pipewire/60-echo-cancel.conf.optional index 8ee46fa..b8a91a2 100644 --- a/pi/pipewire/60-echo-cancel.conf.optional +++ b/pi/pipewire/60-echo-cancel.conf.optional @@ -4,10 +4,12 @@ # # It creates an echo-cancelled mic ("robot_ec_source") and a matching speaker # ("robot_ec_sink"). After enabling it, uncomment the pulse.device lines in the -# Omarchy tunnel config so the tunnels use these nodes instead of the raw ones. +# Omarchy tunnel config (host/omarchy/50-robot-tunnel.conf) so the tunnels use these nodes instead of the +# raw ones. # -# WebRTC echo cancellation costs some CPU; a Pi Zero 2 W manages it at speech rates, -# but check `top` afterwards. +# UNTESTED: it only matters once the speakers are fitted (the robot has none yet). WebRTC echo +# cancellation costs some CPU, and the Pi 4 already runs the whole robot with about 40% of the machine +# idle; check `top` and `robot://status` (temperature) after enabling it. context.modules = [ { name = libpipewire-module-echo-cancel diff --git a/ros/hermes_audio/hermes_audio/audio_state_node.py b/ros/hermes_audio/hermes_audio/audio_state_node.py index 633fe4d..ed2a4a7 100644 --- a/ros/hermes_audio/hermes_audio/audio_state_node.py +++ b/ros/hermes_audio/hermes_audio/audio_state_node.py @@ -4,8 +4,9 @@ listening. Audio itself never goes through ROS; this reads PipeWire with `parec` Publishes: /audio/levels (std_msgs/Float32MultiArray): [mic_dbfs, speaker_dbfs] /audio/activity (std_msgs/String): speaking | listening | quiet (on change, and 2 Hz) /audio/scope/mic, /audio/scope/speaker (std_msgs/Float32MultiArray): a trigger-locked - window of `scope_samples` samples at 16 kHz in [-1, 1], while there is sound: the face - draws the mouth from it, so the trace is the real signal + window of `scope_samples` samples at 16 kHz in [-1, 1]. The mic's is published only while + listening and the speaker's only while speaking (idle, nothing goes out): the face draws the + mouth from it, so the trace is the real signal """ import rclpy from rclpy.node import Node diff --git a/ros/hermes_mcp/hermes_mcp/mcp_node.py b/ros/hermes_mcp/hermes_mcp/mcp_node.py index e7c0c6e..a3c11b9 100644 --- a/ros/hermes_mcp/hermes_mcp/mcp_node.py +++ b/ros/hermes_mcp/hermes_mcp/mcp_node.py @@ -2,7 +2,8 @@ Serves MCP (streamable HTTP, bearer token) to Hermes on the USB link and talks to the graph locally. Subscribes: image_topic (sensor_msgs/CompressedImage, default /camera/image_raw/compressed) - /vision/faces (vision_msgs/Detection2DArray) and /camera/camera_info, for faces() and look() + /vision/faces (vision_msgs/Detection2DArray) and /camera/camera_info, for faces() and look(); + /health/* from hermes_thermal, for the health block of robot://status Publishes: /hermes/state (std_msgs/String): `idle` once a second while an authenticated MCP client is connected: the agent heartbeat the face uses to wake and, when it stops, to fall asleep ("no agent connected"); diff --git a/ros/hermes_vision/hermes_vision/calibrate.py b/ros/hermes_vision/hermes_vision/calibrate.py index 704bce9..f3e29e9 100644 --- a/ros/hermes_vision/hermes_vision/calibrate.py +++ b/ros/hermes_vision/hermes_vision/calibrate.py @@ -4,10 +4,12 @@ COLS x ROWS are the board's *inner corners* (a board of 10 x 7 squares has 9 x 6). With the robot running, hold the board in front of the camera at different distances, tilts and positions, keeping the -whole board in view; it collects views that differ enough from those already taken, solves with OpenCV -and writes ~/.ros/camera_info/_x.yaml (restart the robot to load it). A good result -has a reprojection error under about 0.5 px. Calibration is only valid for the focus it was taken at, -so set the lens focus first, and for the image size in use. +whole board in view, and reach the edges and corners of the image (it says which regions are still +uncovered); it collects views that differ enough from those already taken, solves with OpenCV (k3 held at +zero), drops blurred outlier views, and writes ~/.ros/camera_info/_x.yaml (restart the +robot to load it) plus the collected views as a .npz, so `--from-views FILE` can re-solve them without a +camera. A good result has a reprojection error under about 0.5 px. Calibration is only valid for the focus +it was taken at, so set the lens focus first, and for the image size in use. """ from __future__ import annotations @@ -126,7 +128,7 @@ def main() -> int: p.add_argument('--cols', type=int, required=True, help='inner corners across') p.add_argument('--rows', type=int, required=True, help='inner corners down') p.add_argument('--square-mm', type=float, required=True, help='side of one square, measured') - p.add_argument('--views', type=int, default=20) + p.add_argument('--views', type=int, default=24) p.add_argument('--timeout', type=float, default=240) p.add_argument('--topic', default='/camera/image_raw/compressed') p.add_argument('--name', default=DEFAULT_NAME) diff --git a/ros/hermes_vision/hermes_vision/recognize.py b/ros/hermes_vision/hermes_vision/recognize.py index a7d64ac..ee8da62 100644 --- a/ros/hermes_vision/hermes_vision/recognize.py +++ b/ros/hermes_vision/hermes_vision/recognize.py @@ -107,27 +107,3 @@ def iou(a: Face, b: Face) -> float: return 0.0 inter = iw * ih return inter / (a.w * a.h + b.w * b.h - inter) - - -class IdentityCache: - """Recognition is the expensive step, and a face that has barely moved is the same person: reuse - a tracked face's name for period_s before re-identifying it. Faces match by box overlap.""" - - def __init__(self, period_s: float = 1.0, iou_min: float = 0.4): - self.period_s, self.iou_min = period_s, iou_min - self._entries: list[tuple[Face, str, float]] = [] - - def update(self, faces, now: float, identify) -> list[str]: - """Names for faces (same order). identify(face) -> name is called only for new or stale faces.""" - names, nxt = [], [] - for f in faces: - hit = max(((iou(f, pf), (pf, n, t)) for pf, n, t in self._entries), key=lambda x: x[0], default=None) - if hit and hit[0] >= self.iou_min and now - hit[1][2] < self.period_s: - _, name, t = hit[1] - nxt.append((f, name, t)) # keep the old stamp so it ages and gets rechecked - else: - name = identify(f) - nxt.append((f, name, now)) - names.append(name) - self._entries = nxt - return names diff --git a/ros/hermes_vision/test/test_identity_cache.py b/ros/hermes_vision/test/test_identity_cache.py deleted file mode 100644 index d53572d..0000000 --- a/ros/hermes_vision/test/test_identity_cache.py +++ /dev/null @@ -1,40 +0,0 @@ -from hermes_vision.detect import Face -from hermes_vision.recognize import IdentityCache, iou - - -def test_iou(): - a = Face(0, 0, 100, 100) - assert iou(a, a) == 1.0 - assert iou(a, Face(200, 200, 50, 50)) == 0.0 - assert 0.3 < iou(a, Face(50, 0, 100, 100)) < 0.4 - - -def test_a_tracked_face_is_identified_once_per_period(): - calls = [] - cache = IdentityCache(period_s=1.0) - ident = lambda f: (calls.append(f), 'kyle')[1] - assert cache.update([Face(100, 100, 80, 80)], 0.0, ident) == ['kyle'] - for i in range(1, 8): # the face drifts a little, 4 times a second - assert cache.update([Face(100 + i, 100, 80, 80)], i * 0.125, ident) == ['kyle'] - assert len(calls) == 1 - cache.update([Face(108, 100, 80, 80)], 1.1, ident) # stale: rechecked - assert len(calls) == 2 - - -def test_a_new_face_is_identified_immediately_and_unknown_is_cached_too(): - calls = [] - cache = IdentityCache() - ident = lambda f: (calls.append(f), '' if f.x > 400 else 'kyle')[1] - names = cache.update([Face(100, 100, 80, 80), Face(500, 100, 80, 80)], 0.0, ident) - assert names == ['kyle', ''] and len(calls) == 2 - assert cache.update([Face(101, 100, 80, 80), Face(501, 100, 80, 80)], 0.3, ident) == ['kyle', ''] - assert len(calls) == 2 # the stranger is not re-embedded every frame - - -def test_a_face_that_jumps_is_a_different_face(): - calls = [] - cache = IdentityCache() - ident = lambda f: (calls.append(f), 'kyle')[1] - cache.update([Face(100, 100, 80, 80)], 0.0, ident) - cache.update([Face(600, 300, 80, 80)], 0.1, ident) - assert len(calls) == 2 diff --git a/ros/hermes_vision/test/test_tracking.py b/ros/hermes_vision/test/test_tracking.py index 6251d15..12f160a 100644 --- a/ros/hermes_vision/test/test_tracking.py +++ b/ros/hermes_vision/test/test_tracking.py @@ -53,3 +53,11 @@ def test_a_stranger_is_unknown_not_named(): def test_without_an_identifier_nothing_is_named(): (t,) = FaceTracker().update([f(100)], 0.0) assert t.name == '' and t.id == 1 + + +def test_iou(): + from hermes_vision.recognize import iou + a = Face(0, 0, 100, 100) + assert iou(a, a) == 1.0 + assert iou(a, Face(200, 200, 50, 50)) == 0.0 + assert 0.3 < iou(a, Face(50, 0, 100, 100)) < 0.4