Add hermes_vision, build camera_ros on the fork, and bring up WM8960 audio
hermes_vision wraps OpenCV YuNet (Haar fallback) as a plain module plus a node that publishes /vision/faces as vision_msgs/Detection2DArray. It replaces the standalone detect_faces.py. pi/robot-env.sh puts the Raspberry Pi libcamera fork ahead of ROS's own copy, the venv's OpenCV 4.10 ahead of the apt one, and fixes the ROS domain and subnet-only discovery. camera_ros 0.7.0 now runs against the fork and publishes raw and compressed images. The stock wm8960-soundcard overlay fails on Ubuntu 24.04 with "No MCLK configured": it hangs the clock on the sound card's codec link, but the codec driver reads mclk from its own node. hermes-wm8960.dts adds it there, and capture opens at 8-48 kHz. The microphones still read digital zeros; see CONTEXT.md for what has been ruled out. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AFN8nRRCiDiMN7ChBpHJhs
This commit is contained in:
Claude Sonnet 5.5
@@ -1,101 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detect faces in images and write annotated copies.
|
||||
|
||||
detect_faces.py IMAGE [IMAGE ...] [--out DIR] [--min-size PX]
|
||||
|
||||
Uses OpenCV's YuNet detector when its model loads (~/models/face_detection_yunet_2023mar.onnx),
|
||||
otherwise the Haar cascades shipped with opencv-data (frontal, plus profile in both directions).
|
||||
Prints one line per image: the number of faces, then x,y,w,h boxes in pixels.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
HAAR = '/usr/share/opencv4/haarcascades'
|
||||
YUNET = os.path.expanduser('~/models/face_detection_yunet_2023mar.onnx')
|
||||
|
||||
|
||||
def yunet_detector(w, h):
|
||||
if not os.path.exists(YUNET):
|
||||
return None
|
||||
try:
|
||||
cv2.FaceDetectorYN.create(YUNET, '', (640, 480), 0.6, 0.3, 5000)
|
||||
except cv2.error:
|
||||
return None
|
||||
by_size = {}
|
||||
|
||||
def run(img):
|
||||
# This OpenCV's YuNet needs input sides that are multiples of 32: work at about 640 wide.
|
||||
h0, w0 = img.shape[:2]
|
||||
scale = 640 / max(w0, h0) if max(w0, h0) > 640 else 1.0
|
||||
tw, th = max(32, round(w0 * scale / 32) * 32), max(32, round(h0 * scale / 32) * 32)
|
||||
small = cv2.resize(img, (tw, th), interpolation=cv2.INTER_AREA)
|
||||
if (tw, th) not in by_size: # OpenCV 4.6 breaks if the input size changes after create()
|
||||
by_size[(tw, th)] = cv2.FaceDetectorYN.create(YUNET, '', (tw, th), 0.6, 0.3, 5000)
|
||||
_, faces = by_size[(tw, th)].detect(small)
|
||||
if faces is None:
|
||||
return []
|
||||
fx, fy = w0 / tw, h0 / th
|
||||
return [(int(f[0] * fx), int(f[1] * fy), int(f[2] * fx), int(f[3] * fy)) for f in faces]
|
||||
return run
|
||||
|
||||
|
||||
def haar_detector(min_size):
|
||||
frontal = cv2.CascadeClassifier(os.path.join(HAAR, 'haarcascade_frontalface_alt2.xml'))
|
||||
profile = cv2.CascadeClassifier(os.path.join(HAAR, 'haarcascade_profileface.xml'))
|
||||
|
||||
def run(img):
|
||||
gray = cv2.equalizeHist(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY))
|
||||
ms = (min_size, min_size)
|
||||
boxes = [tuple(b) for b in frontal.detectMultiScale(gray, 1.1, 5, minSize=ms)]
|
||||
for flip in (False, True): # the profile cascade only sees one side
|
||||
g = cv2.flip(gray, 1) if flip else gray
|
||||
for x, y, w, h in profile.detectMultiScale(g, 1.1, 5, minSize=ms):
|
||||
boxes.append((gray.shape[1] - x - w if flip else x, y, w, h))
|
||||
return merge(boxes)
|
||||
return run
|
||||
|
||||
|
||||
def merge(boxes, overlap=0.4):
|
||||
"""Drop boxes that mostly overlap an earlier one (frontal and profile can both fire)."""
|
||||
kept = []
|
||||
for x, y, w, h in boxes:
|
||||
for kx, ky, kw, kh in kept:
|
||||
iw = min(x + w, kx + kw) - max(x, kx)
|
||||
ih = min(y + h, ky + kh) - max(y, ky)
|
||||
if iw > 0 and ih > 0 and iw * ih > overlap * min(w * h, kw * kh):
|
||||
break
|
||||
else:
|
||||
kept.append((int(x), int(y), int(w), int(h)))
|
||||
return kept
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument('images', nargs='+')
|
||||
p.add_argument('--out', default='.')
|
||||
p.add_argument('--min-size', type=int, default=40)
|
||||
args = p.parse_args()
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
detector, name = None, None
|
||||
for path in args.images:
|
||||
img = cv2.imread(path)
|
||||
if img is None:
|
||||
print(f'{path}: unreadable')
|
||||
continue
|
||||
if detector is None:
|
||||
detector = yunet_detector(img.shape[1], img.shape[0])
|
||||
name = 'yunet' if detector else 'haar'
|
||||
detector = detector or haar_detector(args.min_size)
|
||||
boxes = detector(img)
|
||||
for x, y, w, h in boxes:
|
||||
cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2)
|
||||
dest = os.path.join(args.out, 'faces-' + os.path.basename(path).rsplit('.', 1)[0] + '.jpg')
|
||||
cv2.imwrite(dest, img)
|
||||
print(f'{path}: {len(boxes)} face(s) [{name}] {boxes} -> {dest}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# BrainCraft HAT audio (HANDOFF package 1): our WM8960 overlay (pi/overlays/hermes-wm8960.dts: the
|
||||
# stock wm8960-soundcard plus the clock on the codec node, which Ubuntu's driver needs), with the
|
||||
# Pi's own headphone audio off (it shares PWM hardware with the servos). Run as root. Re-runnable.
|
||||
# Needs a reboot afterwards.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
command -v dtc >/dev/null || apt-get -y install device-tree-compiler
|
||||
dtc -@ -I dts -O dtb -o /boot/firmware/overlays/hermes-wm8960.dtbo overlays/hermes-wm8960.dts
|
||||
|
||||
cfg=/boot/firmware/config.txt
|
||||
[ -e "$cfg.bak-audio" ] || cp "$cfg" "$cfg.bak-audio"
|
||||
sed -i 's/^dtparam=audio=on$/dtparam=audio=off/' "$cfg"
|
||||
sed -i '/^dtoverlay=wm8960-soundcard$/d' "$cfg"
|
||||
if ! grep -qx 'dtoverlay=hermes-wm8960' "$cfg"; then
|
||||
# dtparam lines must come before the first dtoverlay, so put this right after vc4-kms-v3d
|
||||
sed -i '0,/^dtoverlay=vc4-kms-v3d$/s//dtoverlay=vc4-kms-v3d\ndtoverlay=hermes-wm8960/' "$cfg"
|
||||
fi
|
||||
grep -nE '^dtparam=audio|^dtoverlay=(hermes-)?wm8960' "$cfg"
|
||||
echo "done; reboot to apply"
|
||||
@@ -0,0 +1,87 @@
|
||||
/dts-v1/;
|
||||
/plugin/;
|
||||
|
||||
/*
|
||||
* BrainCraft HAT audio: the WM8960 codec at I2C 0x1a with its 12.288 MHz master clock.
|
||||
*
|
||||
* Same as the kernel's wm8960-soundcard overlay, except that the clock is also attached to the
|
||||
* codec's own node. Ubuntu 24.04's wm8960 driver reads "mclk" from there; the stock overlay only
|
||||
* puts it on the sound card's codec link, so every capture/playback fails with
|
||||
* "wm8960: No MCLK configured". Compiled by pi/install-audio-overlay.sh.
|
||||
*/
|
||||
/ {
|
||||
compatible = "brcm,bcm2835";
|
||||
|
||||
fragment@0 {
|
||||
target = <&i2s_clk_producer>;
|
||||
__overlay__ {
|
||||
status = "okay";
|
||||
};
|
||||
};
|
||||
|
||||
fragment@1 {
|
||||
target-path = "/";
|
||||
__overlay__ {
|
||||
wm8960_mclk: wm8960_mclk {
|
||||
compatible = "fixed-clock";
|
||||
#clock-cells = <0>;
|
||||
clock-frequency = <12288000>;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
fragment@2 {
|
||||
target = <&i2c1>;
|
||||
__overlay__ {
|
||||
#address-cells = <1>;
|
||||
#size-cells = <0>;
|
||||
status = "okay";
|
||||
|
||||
wm8960: wm8960 {
|
||||
compatible = "wlf,wm8960";
|
||||
reg = <0x1a>;
|
||||
#sound-dai-cells = <0>;
|
||||
clocks = <&wm8960_mclk>;
|
||||
clock-names = "mclk";
|
||||
AVDD-supply = <&vdd_5v0_reg>;
|
||||
DVDD-supply = <&vdd_3v3_reg>;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
fragment@3 {
|
||||
target = <&sound>;
|
||||
slave_overlay: __overlay__ {
|
||||
compatible = "simple-audio-card";
|
||||
simple-audio-card,format = "i2s";
|
||||
simple-audio-card,name = "wm8960-soundcard";
|
||||
status = "okay";
|
||||
|
||||
simple-audio-card,widgets =
|
||||
"Microphone", "Mic Jack",
|
||||
"Line", "Line In",
|
||||
"Line", "Line Out",
|
||||
"Speaker", "Speaker",
|
||||
"Headphone", "Headphone Jack";
|
||||
simple-audio-card,routing =
|
||||
"Headphone Jack", "HP_L",
|
||||
"Headphone Jack", "HP_R",
|
||||
"Speaker", "SPK_LP",
|
||||
"Speaker", "SPK_LN",
|
||||
"LINPUT1", "Mic Jack",
|
||||
"LINPUT3", "Mic Jack",
|
||||
"RINPUT1", "Mic Jack",
|
||||
"RINPUT2", "Mic Jack";
|
||||
|
||||
simple-audio-card,cpu {
|
||||
sound-dai = <&i2s_clk_producer>;
|
||||
};
|
||||
|
||||
dailink0_slave: simple-audio-card,codec {
|
||||
sound-dai = <&wm8960>;
|
||||
clocks = <&wm8960_mclk>;
|
||||
clock-names = "mclk";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
# Source this before running robot nodes on the Pi: source ~/robot-env.sh
|
||||
# - ROS Jazzy, then the overlay workspaces (camera_ros built against the fork; hermes_* packages)
|
||||
# - the Raspberry Pi libcamera fork ahead of ROS's own copy (same soname, but ROS's crashes in the
|
||||
# Pi image plugin: see CONTEXT.md)
|
||||
# - ~/venv's site-packages first: Blinka/Adafruit for the display, OpenCV 4.10 for YuNet
|
||||
# - a fixed ROS domain and subnet-only discovery, matching the host-side podman shell
|
||||
source /opt/ros/jazzy/setup.bash
|
||||
[ -f "$HOME/camera_ws/install/setup.bash" ] && source "$HOME/camera_ws/install/setup.bash"
|
||||
[ -f "$HOME/ros_ws/install/setup.bash" ] && source "$HOME/ros_ws/install/setup.bash"
|
||||
export LD_LIBRARY_PATH="/opt/libcamera-rpi/lib/aarch64-linux-gnu:${LD_LIBRARY_PATH:-}"
|
||||
export PYTHONPATH="$HOME/venv/lib/python3.12/site-packages:${PYTHONPATH:-}"
|
||||
export ROS_DOMAIN_ID="${ROS_DOMAIN_ID:-42}"
|
||||
export ROS_AUTOMATIC_DISCOVERY_RANGE="${ROS_AUTOMATIC_DISCOVERY_RANGE:-SUBNET}"
|
||||
Reference in New Issue
Block a user