kyle/hermes-bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Give Hermes recognitions, not just photos: faces(), tracking ids, discovery

Browse Source 
Hermes asked for structured detections instead of a bare JPEG. faces() returns the
faces in view with no photo: a persistent track id, bbox, detection confidence, how
long the person has been in view, and identity {name, similarity} or null, never a
made-up name. look() returns that same JSON as text plus the photo (send_image can
skip it); list_people() gives names and counts; robot://status and two prompts make
the server discoverable, where list_resources and list_prompts were empty. The
detector now tracks faces across frames (FaceTracker) and publishes the track id and
the identity as a second hypothesis.

Deliberately not built: embeddings in responses, age/gender attributes, auto-updating
face data, and a remote enroll tool (a prompt could make the robot call anyone "Kyle").

Checked end to end: a real Hermes turn on the local ornith model called faces() (0.03 s,
no image) and answered that Kyle was there at 0.78 similarity. Also: the camera is
calibrated (18 of 24 views, 0.296 px, outlier rejection added) and the calibration is
kept in pi/calibration and restored by the installer.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AFN8nRRCiDiMN7ChBpHJhs
This commit is contained in:
kyle
2026-10-05 01:21:32 -07:00
co-authored by Claude Sonnet 5.5
parent 497f026dc9
commit 3b65697f1e
16 changed files with 599 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ros/hermes_mcp/hermes_mcp/bridge.py
+145 -10
View File
@@ -2,16 +2,25 @@
so it is testable with a stub camera.
MCP tool names are the stable contract with Hermes and must not change when the internals do:
look() -> the next fresh JPEG from the camera
look(send_image) -> the faces in view as JSON text, and (by default) the next fresh JPEG
faces() -> the same faces summary with no photo: cheap, the right call for "who is here"
list_people() -> the names enrolled on the robot (never their face data)
turn_head(pan, tilt) -> only registered if a head backend is given (servos fitted)
head_position() -> likewise
Also an `robot://status` resource and prompts, so a client can discover what this robot can do.
Faces never come with embeddings: they are biometric templates, meaningless to a language model, and
the registry stays on the robot. A face that is not recognised has identity null; a name is never
made up. Enrolling people is deliberately not a tool: it is a step the owner takes on the robot.
"""
from __future__ import annotations
import hmac
import json
import os
import threading
import time
from typing import Callable, Optional, Protocol
from typing import Any, Callable, Literal, Optional, Protocol
from mcp.server.mcpserver import Image, MCPServer
from mcp.server.transport_security import TransportSecuritySettings
@@ -43,32 +52,131 @@ class LatestFrame:
return self._jpeg
class FacesBuffer:
"""The newest face detections, with how long each tracked person has been in view."""
def __init__(self) -> None:
self._cv = threading.Condition()
self._faces: Optional[list[dict]] = None
self._size = (1296, 972)
self._t = -1.0
self._first: dict[int, tuple[float, float]] = {} # track id -> (first seen, last seen)
def set_size(self, width: int, height: int) -> None:
with self._cv:
self._size = (int(width), int(height))
def put(self, faces: list[dict], t: Optional[float] = None) -> None:
"""faces: dicts with track_id, bbox {x, y, w, h}, confidence, name ('' if unknown), similarity."""
now = time.monotonic() if t is None else t
with self._cv:
for f in faces:
first, _ = self._first.get(f['track_id'], (now, now))
self._first[f['track_id']] = (first, now)
self._first = {i: v for i, v in self._first.items() if now - v[1] < 5.0}
self._faces, self._t = faces, now
self._cv.notify_all()
def _snapshot(self, now: float) -> dict:
w, h = self._size
out = []
for f in self._faces or []:
b = f['bbox']
first = self._first.get(f['track_id'], (now, now))[0]
out.append({
'track_id': f['track_id'],
'bbox': {k: round(float(b[k])) for k in ('x', 'y', 'w', 'h')},
'bbox_norm': {'cx': round((b['x'] + b['w'] / 2) / w, 3), 'cy': round((b['y'] + b['h'] / 2) / h, 3),
'w': round(b['w'] / w, 3), 'h': round(b['h'] / h, 3)},
'confidence': round(float(f['confidence']), 2),
'identity': ({'name': f['name'], 'similarity': round(float(f['similarity']), 2)}
if f.get('name') else None),
'seen_for_s': round(max(0.0, now - first), 1),
})
return {'available': True, 'count': len(out), 'image_size': {'w': w, 'h': h},
'data_age_s': round(max(0.0, now - self._t), 2), 'faces': out}
def summary(self, max_age_s: float = 1.5, wait_s: float = 0.0) -> dict:
"""The faces in view now. If the newest data is older than max_age_s, wait up to wait_s for fresh
data; if there is none, say so rather than reporting a stale room."""
deadline = time.monotonic() + wait_s
with self._cv:
while self._faces is None or time.monotonic() - self._t > max_age_s:
remaining = deadline - time.monotonic()
if remaining <= 0:
return {'available': False, 'reason': 'face detection is not producing results right now'}
self._cv.wait(remaining)
return self._snapshot(time.monotonic())
def people_summary(path: str) -> dict:
"""Who is enrolled on this robot: names and how many samples, read straight from the registry file
(JSON), never the face data itself."""
try:
with open(path) as f:
db = json.load(f)
except FileNotFoundError:
db = {}
return {'count': len(db), 'people': [{'name': n, 'samples': len(v)} for n, v in sorted(db.items())]}
class Head(Protocol):
def move(self, pan: float, tilt: float) -> tuple[float, float]: ...
def position(self) -> tuple[float, float]: ...
def build_mcp(frames: LatestFrame, announce_look: Callable[[], None] = lambda: None,
head: Optional[Head] = None, look_timeout_s: float = 3.0) -> MCPServer:
head: Optional[Head] = None, look_timeout_s: float = 3.0,
faces_buf: Optional[FacesBuffer] = None, people_path: str = '',
hardware: Optional[dict] = None) -> MCPServer:
faces_buf = faces_buf or FacesBuffer()
hw = {'camera': True, 'head': head is not None, 'speakers': False, **(hardware or {})}
mcp = MCPServer(
"robot",
instructions=(
"This server is a small physical robot on the user's desk. "
"Call look() whenever you need to see what is in front of it; the image is current. "
"If turn_head is available, aim first and then look."
"This server is a small physical robot on the user's desk, with a camera and a face display. "
"To know WHO or HOW MANY people are there, call faces(): it is instant and returns names, "
"positions and tracking ids with no photo; identity null means unknown, so say so rather "
"than guessing. Call look() when you need to see the scene itself: it returns the faces "
"summary and a current photo. list_people() says who the robot knows. "
+ ("turn_head aims the camera: aim first, then look. " if head is not None else
"The camera is fixed: there is no head to turn. ")
+ "Only faces are detected: not other people-shaped things, objects or motion. "
"Read robot://status for what hardware this robot has."
),
)
@mcp.tool()
def look() -> Image:
"""Take a photograph with the robot's camera and return it. Use this to see the room,
the desk, the person, or anything the user asks you to look at."""
def faces() -> dict[str, Any]:
"""Report the faces the robot's camera sees right now, with no photo (instant and cheap).
Each face has a track_id (the same person keeps the same id while they stay in view),
a bounding box in camera pixels and normalised, a detection confidence, seen_for_s (how long
they have been in view), and identity: {name, similarity} if the robot recognises them, or
null if not. null means unknown: never invent a name for it. similarity is a cosine score, not a
probability. Use this to answer who or how many."""
return faces_buf.summary(max_age_s=1.5, wait_s=1.5)
@mcp.tool()
def look(send_image: Literal["always", "never"] = "always") -> list:
"""Look through the robot's camera. Returns the faces summary (JSON text, same as faces()) and,
unless send_image is "never", a current photograph of the scene. Use it to see the room, the
desk, a document, or anything the user asks you to look at; use faces() alone for who/how many."""
t0 = time.monotonic()
announce_look() # the face squints and flashes
summary = faces_buf.summary(max_age_s=1.5, wait_s=0.3)
text = json.dumps({'faces': summary})
if send_image == "never":
return [text]
jpeg = frames.wait_newer(t0, look_timeout_s)
if jpeg is None:
raise RuntimeError("the robot's camera is not producing images")
return Image(data=jpeg, format="jpeg")
return [text, Image(data=jpeg, format="jpeg")]
@mcp.tool()
def list_people() -> dict[str, Any]:
"""Who the robot has been taught to recognise: names and how many samples each, never their
face data. People are enrolled by the owner on the robot, not through this server."""
return people_summary(people_path) if people_path else {'count': 0, 'people': []}
if head is not None:
@mcp.tool()
@@ -86,6 +194,33 @@ def build_mcp(frames: LatestFrame, announce_look: Callable[[], None] = lambda: N
p, t = head.position()
return f"pan {p:+.0f} degrees, tilt {t:+.0f} degrees"
@mcp.resource("robot://status")
def status() -> str:
"""What this robot is and what it can do right now: hardware present, what is detected, who is known."""
live = faces_buf.summary(max_age_s=3.0)
return json.dumps({
'robot': 'hermes-bot', 'hardware': hw,
'camera': {'image_size': live.get('image_size') if live['available'] else None},
'detects': ['faces'], 'not_detected': ['people', 'objects', 'motion'],
'face_detection_running': live['available'],
'recognition': {'enrolled': people_summary(people_path)['count'] if people_path else 0,
'registry': 'on the robot; enrolled by the owner, not through MCP; embeddings never leave it'},
'tools': sorted(['faces', 'look', 'list_people'] + (['turn_head', 'head_position'] if head else [])),
}, indent=2)
@mcp.prompt()
def who_is_here() -> str:
"""Find out who is at the robot's desk."""
return ("Call faces() and tell me who is there. Name the people the robot recognises, say how many "
"faces it does not recognise (identity null: do not guess who they are), and mention anyone "
"who has only just appeared. If face detection is unavailable, say so.")
@mcp.prompt()
def look_around() -> str:
"""See the scene and describe it."""
return ("Call look() and describe what you see in the photo, then say who the faces summary says is "
"there. Do not name anyone the summary lists as identity null.")
return mcp
ros/hermes_mcp/hermes_mcp/mcp_node.py
+30 -3
View File
@@ -2,6 +2,7 @@
Serves MCP (streamable HTTP, bearer token) to Hermes on the USB link and talks to the graph locally.
Subscribes: image_topic (sensor_msgs/CompressedImage, default /camera/image_raw/compressed)
/vision/faces (vision_msgs/Detection2DArray) and /camera/camera_info, for faces() and look()
Publishes: /hermes/state (std_msgs/String):
`idle` once a second while an authenticated MCP client is connected: the agent heartbeat
the face uses to wake and, when it stops, to fall asleep ("no agent connected");
@@ -16,10 +17,11 @@ import time
import rclpy
import uvicorn
from rclpy.node import Node
from sensor_msgs.msg import CompressedImage
from sensor_msgs.msg import CameraInfo, CompressedImage
from std_msgs.msg import String
from vision_msgs.msg import Detection2DArray
from .bridge import LatestFrame, build_app, build_mcp
from .bridge import FacesBuffer, LatestFrame, build_app, build_mcp
class McpNode(Node):
@@ -36,6 +38,11 @@ class McpNode(Node):
self.state_pub = self.create_publisher(String, '/hermes/state', 10)
self.frames = LatestFrame()
self.create_subscription(CompressedImage, topic, lambda m: self.frames.put(bytes(m.data)), 1)
self.faces = FacesBuffer()
self.create_subscription(Detection2DArray, '/vision/faces', self._on_faces, 5)
self.create_subscription(CameraInfo, '/camera/camera_info', self._on_camera_info, 1)
people_path = os.path.expanduser(str(param('faces_db', '~/.local/share/hermes/faces.json')))
hardware = {'speakers': bool(param('speakers', False))}
token = os.environ.get('ROBOT_MCP_TOKEN', '').strip()
self.activity = None
@@ -45,11 +52,31 @@ class McpNode(Node):
self.get_logger().error('ROBOT_MCP_TOKEN is missing or too short (openssl rand -hex 32): '
'not serving MCP, and no heartbeat')
return
mcp = build_mcp(self.frames, self._announce_look, head=None, look_timeout_s=look_timeout)
mcp = build_mcp(self.frames, self._announce_look, head=None, look_timeout_s=look_timeout,
faces_buf=self.faces, people_path=people_path, hardware=hardware)
self.app, self.activity = build_app(mcp, token, self.host)
threading.Thread(target=self._serve, name='mcp-http', daemon=True).start()
self.create_timer(1.0, self._heartbeat)
def _on_camera_info(self, msg: CameraInfo) -> None:
if msg.width and msg.height:
self.faces.set_size(msg.width, msg.height)
def _on_faces(self, msg: Detection2DArray) -> None:
faces = []
for d in msg.detections:
if not d.results:
continue
c = d.bbox.center.position
faces.append({
'track_id': int(d.id) if d.id.isdigit() else hash(d.id) & 0xFFFF,
'bbox': {'x': c.x - d.bbox.size_x / 2, 'y': c.y - d.bbox.size_y / 2, 'w': d.bbox.size_x, 'h': d.bbox.size_y},
'confidence': d.results[0].hypothesis.score,
'name': d.results[1].hypothesis.class_id if len(d.results) > 1 else '',
'similarity': d.results[1].hypothesis.score if len(d.results) > 1 else 0.0,
})
self.faces.put(faces)
def _announce_look(self) -> None:
self.state_pub.publish(String(data='looking'))
ros/hermes_mcp/package.xml
+1
View File
@@ -10,6 +10,7 @@
<depend>rclpy</depend>
<depend>sensor_msgs</depend>
<depend>std_msgs</depend>
<depend>vision_msgs</depend>
<test_depend>python3-pytest</test_depend>
ros/hermes_mcp/test/test_bridge.py
+80 -5
View File
@@ -1,5 +1,6 @@
import asyncio
import base64
import json
import threading
import time
@@ -8,7 +9,8 @@ import pytest
pytest.importorskip('mcp')
from starlette.testclient import TestClient # noqa: E402
from hermes_mcp.bridge import Activity, BearerAuth, LatestFrame, build_app, build_mcp # noqa: E402
from hermes_mcp.bridge import (Activity, BearerAuth, FacesBuffer, LatestFrame, build_app, build_mcp, # noqa: E402
people_summary)
TOKEN = 'x' * 32
@@ -25,9 +27,11 @@ def tool_names(mcp):
return {t.name for t in asyncio.run(mcp.list_tools())}
def test_tools_are_look_only_without_a_head_and_keep_their_names_with_one():
assert tool_names(build_mcp(LatestFrame())) == {'look'}
assert tool_names(build_mcp(LatestFrame(), head=FakeHead())) == {'look', 'turn_head', 'head_position'}
def test_tools_have_no_head_tools_without_a_head_and_keep_their_names_with_one():
base = {'look', 'faces', 'list_people'}
assert tool_names(build_mcp(LatestFrame())) == base
assert tool_names(build_mcp(LatestFrame(), head=FakeHead())) == base | {'turn_head', 'head_position'}
assert 'enroll' not in tool_names(build_mcp(LatestFrame(), head=FakeHead())) # enrolment is the owner's step
def test_look_returns_the_next_frame_not_a_stale_one():
@@ -37,8 +41,9 @@ def test_look_returns_the_next_frame_not_a_stale_one():
mcp = build_mcp(frames, announce_look=lambda: announced.append(1), look_timeout_s=2.0)
threading.Timer(0.3, lambda: frames.put(b'\xff\xd8fresh')).start()
result = asyncio.run(mcp.call_tool('look', {}))
image = result.content[0]
text, image = result.content
assert announced == [1]
assert text.type == 'text' and json.loads(text.text)['faces']['available'] is False # no detector running here
assert image.type == 'image' and image.mime_type == 'image/jpeg'
assert base64.b64decode(image.data) == b'\xff\xd8fresh' # the frame that arrived after the call
@@ -97,3 +102,73 @@ def test_bearer_auth_passes_lifespan_through():
asyncio.run(BearerAuth(inner, TOKEN)({'type': 'lifespan'}, None, None))
assert seen == ['lifespan']
def face(track_id, x, name='', sim=0.0, conf=0.9, w=300, h=400):
return {'track_id': track_id, 'bbox': {'x': x, 'y': 100, 'w': w, 'h': h}, 'confidence': conf, 'name': name, 'similarity': sim}
def call(mcp, name, args=None):
return asyncio.run(mcp.call_tool(name, args or {}))
def test_faces_reports_identity_null_for_a_stranger_and_never_makes_up_a_name():
buf = FacesBuffer()
buf.put([face(1, 100, 'Kyle', 0.82), face(2, 600)])
out = call(build_mcp(LatestFrame(), faces_buf=buf), 'faces').structured_content
assert out['available'] and out['count'] == 2 and out['image_size'] == {'w': 1296, 'h': 972}
kyle, stranger = out['faces']
assert kyle['identity'] == {'name': 'Kyle', 'similarity': 0.82}
assert stranger['identity'] is None # unknown, not guessed
assert set(kyle) == {'track_id', 'bbox', 'bbox_norm', 'confidence', 'identity', 'seen_for_s'}
assert 'embedding' not in json.dumps(out)
def test_track_ids_persist_and_seen_for_grows():
buf = FacesBuffer()
buf.put([face(7, 100)], t=time.monotonic() - 3.0)
buf.put([face(7, 105)])
(f,) = buf.summary()['faces']
assert f['track_id'] == 7 and 2.5 < f['seen_for_s'] < 3.6
def test_faces_says_when_detection_is_not_running_instead_of_reporting_an_empty_room():
out = call(build_mcp(LatestFrame(), faces_buf=FacesBuffer()), 'faces').structured_content
assert out['available'] is False and 'reason' in out
buf = FacesBuffer()
buf.put([face(1, 100)], t=time.monotonic() - 30) # stale data is not "now"
assert buf.summary(max_age_s=1.5)['available'] is False
def test_an_empty_room_is_available_with_zero_faces():
buf = FacesBuffer()
buf.put([])
assert buf.summary()['available'] and buf.summary()['count'] == 0
def test_look_can_skip_the_photo():
frames, buf = LatestFrame(), FacesBuffer()
buf.put([face(1, 100, 'Kyle', 0.8)])
frames.put(b'\xff\xd8x')
result = call(build_mcp(frames, faces_buf=buf), 'look', {'send_image': 'never'})
assert len(result.content) == 1 and result.content[0].type == 'text'
assert json.loads(result.content[0].text)['faces']['faces'][0]['identity']['name'] == 'Kyle'
def test_list_people_gives_names_and_counts_never_face_data(tmp_path):
path = tmp_path / 'faces.json'
path.write_text(json.dumps({'Kyle': [[0.1] * 128] * 10, 'Alex': [[0.2] * 128] * 3}))
out = call(build_mcp(LatestFrame(), people_path=str(path)), 'list_people').structured_content
assert out == {'count': 2, 'people': [{'name': 'Alex', 'samples': 3}, {'name': 'Kyle', 'samples': 10}]}
assert people_summary(str(tmp_path / 'missing.json')) == {'count': 0, 'people': []}
def test_the_server_can_be_discovered_resources_and_prompts_are_not_empty():
mcp = build_mcp(LatestFrame(), hardware={'speakers': False})
assert [str(r.uri) for r in asyncio.run(mcp.list_resources())] == ['robot://status']
assert {p.name for p in asyncio.run(mcp.list_prompts())} == {'who_is_here', 'look_around'}
status = json.loads(asyncio.run(mcp.read_resource('robot://status'))[0].content)
assert status['hardware'] == {'camera': True, 'head': False, 'speakers': False}
assert status['detects'] == ['faces'] and 'people' in status['not_detected']
assert 'embeddings never leave it' in status['recognition']['registry']
assert 'turn_head' not in status['tools']
ros/hermes_mcp/test/test_node.py
+16
View File
@@ -0,0 +1,16 @@
import pytest
rclpy = pytest.importorskip('rclpy')
def test_the_node_constructs_without_a_token_and_does_not_serve(monkeypatch):
"""Declaring a parameter twice raises at construction, and nothing else exercises the node."""
monkeypatch.delenv('ROBOT_MCP_TOKEN', raising=False)
from hermes_mcp.mcp_node import McpNode
rclpy.init()
try:
node = McpNode()
assert node.activity is None # no token: not serving, no heartbeat
node.destroy_node()
finally:
rclpy.shutdown()