feat(bus): tines-bus transport and daemon over a host IPC bus

This commit is contained in:
2026-10-24 23:00:37 -07:00
parent 01373d809d
commit ee87808b05
10 changed files with 1036 additions and 1 deletions
+16
View File
@@ -0,0 +1,16 @@
//! The tines bus daemon: routes envelopes between connected clients on a
//! host IPC socket. Usage: `tines-busd <socket-path>`.
fn main() {
let socket = match std::env::args().nth(1) {
Some(s) => s,
None => {
eprintln!("usage: tines-busd <socket-path>");
std::process::exit(2);
}
};
if let Err(e) = tines_bus::daemon::serve(&socket) {
eprintln!("tines-busd: {e}");
std::process::exit(1);
}
}
+309
View File
@@ -0,0 +1,309 @@
//! The bus daemon: a single-threaded, nonblocking hub that routes envelopes
//! between connected clients.
//!
//! # Protocol
//!
//! Every message is `[u32 LE length][payload]`, the length counting payload
//! bytes only (bus-local; the core frames inside are untouched).
//!
//! - **Hello** (client→daemon, first message): payload = 16-byte `NodeId`.
//! A hello for an already-registered id replaces the old connection (the
//! old socket is closed).
//! - **Data** (client→daemon): payload = `[16B from][16B to][frame]`. The
//! daemon drops (and counts on stderr) any message whose `from` is not that
//! socket's registered id, or whose `to` is not registered; otherwise it
//! forwards the envelope verbatim to `to`'s socket. Frames longer than
//! [`MAX_FRAME`] close the connection.
//! - **Control** (daemon→client): payload = `[1B kind][16B NodeId]`; kind
//! `0 = joined`, `1 = left`; sent to every other registered client on join,
//! and to all remaining on leave.
//!
//! EOF or an error from a client's socket is a leave plus close; the daemon
//! itself stays up. No frame content is decoded beyond the routing header.
use std::io::{self, ErrorKind, Read, Write};
use std::os::unix::net::{UnixListener, UnixStream};
use std::path::Path;
use std::time::Duration;
use tines_core::consts::MAX_FRAME;
use tines_core::NodeId;
/// Routing-header size (from + to) of a data envelope.
const HEADER_BYTES: usize = 32;
/// Longest envelope payload the daemon will buffer or forward.
const MAX_PAYLOAD: usize = MAX_FRAME + HEADER_BYTES;
fn id_of(bytes: &[u8]) -> NodeId {
let mut id = [0u8; 16];
id.copy_from_slice(bytes);
id
}
fn control(kind: u8, id: NodeId) -> Vec<u8> {
let mut env = Vec::with_capacity(4 + 17);
env.extend_from_slice(&17u32.to_le_bytes());
env.push(kind);
env.extend_from_slice(&id);
env
}
/// Read one envelope's payload from `s`; `Ok(None)` means clean EOF.
pub(crate) fn read_envelope(s: &mut UnixStream) -> io::Result<Option<Vec<u8>>> {
let mut len_buf = [0u8; 4];
let n = s.read(&mut len_buf)?;
if n == 0 {
return Ok(None);
}
if n != 4 {
return Err(io::Error::new(
ErrorKind::InvalidData,
"short envelope length header",
));
}
let len = u32::from_le_bytes(len_buf) as usize;
if len > MAX_PAYLOAD {
return Err(io::Error::new(
ErrorKind::InvalidData,
"envelope over the size cap",
));
}
let mut payload = vec![0u8; len];
s.read_exact(&mut payload)?;
Ok(Some(payload))
}
struct Conn {
id: Option<NodeId>,
stream: UnixStream,
buf: Vec<u8>,
/// Set to evict this connection on the next apply pass.
close_pending: bool,
}
/// A queued action from processing one message.
enum Action {
/// Close the connection at `idx`; if it was registered, announce a leave.
Close { idx: usize },
/// Write a prepared envelope to the connection at `to`.
Forward { to: usize, env: Vec<u8> },
}
/// Serve the bus on `socket` until the process dies.
///
/// A stale socket file from a previous run is removed first so `bind` cannot
/// fail on it.
pub fn serve(socket: impl AsRef<Path>) -> io::Result<()> {
let socket = socket.as_ref().to_path_buf();
let _ = std::fs::remove_file(&socket);
let listener = UnixListener::bind(&socket)?;
listener.set_nonblocking(true)?;
let mut conns: Vec<Conn> = Vec::new();
loop {
// Accept everything pending.
loop {
match listener.accept() {
Ok((stream, _)) => {
stream.set_nonblocking(true)?;
conns.push(Conn {
id: None,
stream,
buf: Vec::new(),
close_pending: false,
});
}
Err(ref e) if e.kind() == ErrorKind::WouldBlock => break,
Err(_) => break,
}
}
// Read + parse each connection, queueing actions.
let mut actions: Vec<Action> = Vec::new();
for i in 0..conns.len() {
let (closed, msgs) = {
let c = &mut conns[i];
let mut closed = false;
let mut chunk = [0u8; 4096];
loop {
match c.stream.read(&mut chunk) {
Ok(0) => {
closed = true;
break;
}
Ok(n) => {
c.buf.extend_from_slice(&chunk[..n]);
if n < chunk.len() {
break;
}
}
Err(ref e) if e.kind() == ErrorKind::WouldBlock => break,
Err(_) => {
closed = true;
break;
}
}
}
// Split complete envelopes out of the buffer.
let mut msgs = Vec::new();
loop {
if c.buf.len() < 4 {
break;
}
let len = u32::from_le_bytes([c.buf[0], c.buf[1], c.buf[2], c.buf[3]]) as usize;
if len > MAX_PAYLOAD {
// Oversized envelope: protocol violation.
closed = true;
break;
}
if c.buf.len() < 4 + len {
break;
}
msgs.push(c.buf[4..4 + len].to_vec());
c.buf.drain(..4 + len);
}
(closed, msgs)
};
// Process complete messages even when the connection is being
// closed: a burst hello+oversized-data must still register the
// id so the leave is announced to the right peers.
for msg in msgs {
process_msg(i, &msg, &mut conns, &mut actions);
}
if closed {
actions.push(Action::Close { idx: i });
}
}
apply_actions(&mut actions, &mut conns);
std::thread::sleep(Duration::from_millis(2));
}
}
/// Handle one envelope from the connection at `idx`.
fn process_msg(idx: usize, payload: &[u8], conns: &mut [Conn], actions: &mut Vec<Action>) {
let (before, rest) = conns.split_at_mut(idx);
let (me, after) = rest.split_at_mut(1);
let me = &mut me[0];
match me.id {
None => {
// Unregistered sockets may only hello.
if payload.len() != 16 {
actions.push(Action::Close { idx });
return;
}
let id = id_of(payload);
// A hello for an already-registered id replaces the old
// connection (silently: peers never see a flap).
let is_new = !before.iter().chain(after.iter()).any(|c| c.id == Some(id));
if !is_new {
for c in before.iter_mut().chain(after.iter_mut()) {
if c.id == Some(id) {
c.close_pending = true;
}
}
}
me.id = Some(id);
if is_new {
let env = control(0, id);
for (j, c) in before
.iter()
.enumerate()
.chain(after.iter().enumerate().map(|(k, c)| (k + idx + 1, c)))
{
if c.id.is_some() {
actions.push(Action::Forward {
to: j,
env: env.clone(),
});
}
}
}
}
Some(my_id) => {
if payload.len() == 16 {
// Repeat hello on an already-registered socket: no-op.
return;
}
if payload.len() < 32 {
actions.push(Action::Close { idx });
return;
}
let from = id_of(&payload[0..16]);
let to = id_of(&payload[16..32]);
let frame_len = payload.len() - 32;
if from != my_id {
// The daemon certifies `from`: never route a forgery.
eprintln!("tines-busd: dropped: from mismatch on registered socket");
return;
}
if frame_len > MAX_FRAME {
actions.push(Action::Close { idx });
return;
}
// `position()` on the chain is the actual index only for peers
// before the sender; after the sender it is one short.
let to_idx = before.iter().position(|c| c.id == Some(to)).or_else(|| {
after
.iter()
.position(|c| c.id == Some(to))
.map(|k| k + idx + 1)
});
match to_idx {
Some(j) => {
let mut env = Vec::with_capacity(4 + payload.len());
env.extend_from_slice(&(payload.len() as u32).to_le_bytes());
env.extend_from_slice(payload);
actions.push(Action::Forward { to: j, env });
}
None => eprintln!("tines-busd: dropped: unknown destination"),
}
}
}
}
/// Run queued actions: forwards first, then closes (announcing leaves).
fn apply_actions(actions: &mut [Action], conns: &mut Vec<Conn>) {
// Forwards (only to live, registered connections).
for a in actions.iter() {
if let Action::Forward { to, env } = a {
if let Some(c) = conns.get_mut(*to) {
if !c.close_pending && c.id.is_some() && c.stream.write_all(env).is_err() {
c.close_pending = true;
}
}
}
}
// Closes. A registered connection that leaves (EOF, error, protocol
// violation) is announced to everyone remaining; an evicted duplicate
// already has `close_pending` set and is silent.
let mut announced: Vec<NodeId> = Vec::new();
for a in actions.iter() {
if let Action::Close { idx } = a {
if let Some(c) = conns.get_mut(*idx) {
c.close_pending = true;
if let Some(id) = c.id {
announced.push(id);
}
}
}
}
if !announced.is_empty() {
let envs: Vec<Vec<u8>> = announced.iter().map(|id| control(1, *id)).collect();
for env in envs {
for c in conns.iter_mut() {
if c.close_pending || c.id.is_none() {
continue;
}
if c.stream.write_all(&env).is_err() {
c.close_pending = true;
}
}
}
}
conns.retain(|c| !c.close_pending);
}
+13
View File
@@ -0,0 +1,13 @@
//! The tines bus transport: exchange wire frames between nodes over a host
//! IPC bus (Unix sockets), identity-preserving (`Addr = NodeId`).
//!
//! [`BusTransport`] is the tines [`Transport`] implementation; [`daemon`] is
//! the hub that routes frames between all connected nodes. The two sides of
//! a fleet talk through one shared daemon socket.
#![deny(missing_docs)]
pub mod daemon;
mod transport;
pub use transport::{BusError, BusEvent, BusLink, BusTransport, DaemonLink};
+285
View File
@@ -0,0 +1,285 @@
//! The bus transport: a [`Transport`] whose peers are other tines nodes
//! connected to the same host bus daemon (see [`crate::daemon`]).
//!
//! The pure state machine ([`BusTransport`]) is generic over a narrow I/O
//! seam, [`BusLink`], mirroring how `tines-bt` splits logic from radio. The
//! real link ([`DaemonLink`]) is a thread-driven client of the bus daemon;
//! it lives in this crate but is not part of the pure logic.
use std::io::{self, Write};
use std::path::Path;
use std::sync::mpsc;
use tines_core::consts::MAX_FRAME;
use tines_core::{frame_sender, Event, FrameBuf, SendError, Transport};
use crate::daemon;
pub use tines_core::NodeId;
/// An event observed on the bus link.
#[derive(Debug)]
pub enum BusEvent {
/// A peer registered with the daemon.
Joined(NodeId),
/// A registered peer left (or its connection was replaced).
Left(NodeId),
/// A data envelope arrived, attributed to the daemon-certified sender.
Frame(NodeId, Box<FrameBuf>),
/// The link itself closed (daemon gone, socket error, or local shutdown).
Closed,
}
/// Errors a [`BusLink`] can report from `send`.
#[derive(Debug)]
pub enum BusError {
/// The underlying socket/channel failed.
Io(io::Error),
/// The destination is not reachable on this link.
Unreachable,
}
/// The narrow I/O seam behind [`BusTransport`].
///
/// `send` receives the *frame bytes only*; the link stamps the envelope with
/// the sender's identity, so a caller cannot forge who a frame is from.
pub trait BusLink {
/// Poll for the next link event, if any is ready.
fn poll(&mut self) -> Option<BusEvent>;
/// Queue `data` for delivery to `to`.
fn send(&mut self, to: &NodeId, data: &[u8]) -> Result<(), BusError>;
/// Largest frame this link can carry (its transport MTU).
fn mtu(&self) -> usize;
}
/// A thread-driven client of the bus daemon.
///
/// A writer thread owns the socket (sending the hello first, then queued
/// envelopes) and a reader thread owns a clone, decoding inbound envelopes
/// into [`BusEvent`]s. Dropping the link drops the writer's queue, which
/// shuts the socket down, which unblocks the reader — no explicit shutdown
/// protocol needed.
#[derive(Debug)]
pub struct DaemonLink {
me: NodeId,
out_tx: mpsc::Sender<Vec<u8>>,
ev_rx: mpsc::Receiver<BusEvent>,
}
impl DaemonLink {
/// Connect to the daemon at `socket` and hello as `me`.
///
/// The connect retries briefly because a freshly spawned daemon binds
/// asynchronously.
pub fn connect(me: NodeId, socket: &Path) -> io::Result<DaemonLink> {
let mut stream = retry_connect(socket)?;
let mut reader = stream.try_clone()?;
let (out_tx, out_rx) = mpsc::channel::<Vec<u8>>();
let (ev_tx, ev_rx) = mpsc::channel::<BusEvent>();
// Writer thread: hello first, then queued envelopes. On queue
// disconnect (the transport was dropped) it shuts the socket down,
// which also unblocks the reader thread.
let mut hello = Vec::with_capacity(4 + 16);
hello.extend_from_slice(&16u32.to_le_bytes());
hello.extend_from_slice(&me);
std::thread::spawn(move || {
if stream.write_all(&hello).is_err() {
let _ = stream.shutdown(std::net::Shutdown::Both);
return;
}
while let Ok(env) = out_rx.recv() {
if stream.write_all(&env).is_err() {
break;
}
}
let _ = stream.shutdown(std::net::Shutdown::Both);
});
// Reader thread: decode envelopes into events.
std::thread::spawn(move || {
loop {
let payload = match daemon::read_envelope(&mut reader) {
Ok(Some(p)) => p,
Ok(None) | Err(_) => {
let _ = ev_tx.send(BusEvent::Closed);
return;
}
};
let ev = match payload.len() {
17 => match payload[0] {
0 => BusEvent::Joined(id_of(&payload[1..17])),
_ => BusEvent::Left(id_of(&payload[1..17])),
},
n if n >= 32 => {
let from = id_of(&payload[0..16]);
let frame = match FrameBuf::from_slice(&payload[32..]) {
Ok(f) => f,
// Overlong frame: protocol violation, treat as
// a dead link.
Err(_) => {
let _ = ev_tx.send(BusEvent::Closed);
return;
}
};
BusEvent::Frame(from, Box::new(frame))
}
_ => continue, // unknown message shape: ignore
};
if ev_tx.send(ev).is_err() {
return;
}
}
});
Ok(DaemonLink { me, out_tx, ev_rx })
}
}
impl BusLink for DaemonLink {
fn poll(&mut self) -> Option<BusEvent> {
self.ev_rx.try_recv().ok()
}
fn send(&mut self, to: &NodeId, data: &[u8]) -> Result<(), BusError> {
// Envelope: [u32 LE payload len][from][to][frame].
let mut env = Vec::with_capacity(4 + 32 + data.len());
env.extend_from_slice(&((32 + data.len()) as u32).to_le_bytes());
env.extend_from_slice(&self.me);
env.extend_from_slice(to);
env.extend_from_slice(data);
self.out_tx
.send(env)
.map_err(|_| BusError::Io(io::Error::new(io::ErrorKind::BrokenPipe, "bus link closed")))
}
fn mtu(&self) -> usize {
MAX_FRAME
}
}
/// Connect to `socket`, retrying briefly while a just-spawned daemon binds.
fn retry_connect(socket: &Path) -> io::Result<std::os::unix::net::UnixStream> {
let mut last_err = None;
for _ in 0..40 {
match std::os::unix::net::UnixStream::connect(socket) {
Ok(s) => return Ok(s),
Err(e) => last_err = Some(e),
}
std::thread::sleep(std::time::Duration::from_millis(5));
}
Err(last_err.unwrap_or_else(|| {
io::Error::new(io::ErrorKind::ConnectionRefused, "bus daemon never bound")
}))
}
fn id_of(bytes: &[u8]) -> NodeId {
let mut id = [0u8; 16];
id.copy_from_slice(bytes);
id
}
/// The bus transport: tines `Transport` over a [`BusLink`].
///
/// `type Addr = NodeId` — the daemon certifies each message's sender, so bus
/// peer identity is as strong as the other transports'.
pub struct BusTransport<L: BusLink = DaemonLink> {
me: NodeId,
allowlist: Option<NodeId>,
peers: Vec<NodeId>,
link: L,
closed: bool,
}
impl BusTransport<DaemonLink> {
/// Create a bus transport connected to the daemon at `socket`.
///
/// `allowlist` pins the one peer this transport may see and reach
/// (`None` = every bus peer), mirroring the single-peer allowlist of the
/// bt transport.
pub fn new(
me: NodeId,
allowlist: Option<NodeId>,
socket: impl AsRef<Path>,
) -> io::Result<BusTransport<DaemonLink>> {
let link = DaemonLink::connect(me, socket.as_ref())?;
Ok(BusTransport {
me,
allowlist,
peers: Vec::new(),
link,
closed: false,
})
}
}
impl<L: BusLink> BusTransport<L> {
/// Whether `peer` passes the allowlist (and is not ourselves).
fn allows(&self, peer: &NodeId) -> bool {
*peer != self.me && self.allowlist.is_none_or(|a| a == *peer)
}
}
impl<L: BusLink> Transport for BusTransport<L> {
type Addr = NodeId;
fn poll(&mut self) -> Option<Event<NodeId>> {
loop {
match self.link.poll()? {
BusEvent::Joined(addr) => {
if !self.allows(&addr) || self.peers.contains(&addr) {
continue;
}
self.peers.push(addr);
return Some(Event::Contact(addr));
}
BusEvent::Left(addr) => {
if let Some(pos) = self.peers.iter().position(|p| *p == addr) {
self.peers.remove(pos);
return Some(Event::Lost(addr));
}
}
BusEvent::Frame(peer, frame) => {
if !self.allows(&peer) {
continue;
}
// The daemon certifies the envelope's `from`; the frame's
// own declared sender must agree with it.
if frame_sender(frame.as_slice()) != Some(peer) {
continue;
}
return Some(Event::Frame(peer, *frame));
}
BusEvent::Closed => {
if self.closed {
return None;
}
self.closed = true;
// Every bound peer is gone; drain one `Lost` per poll.
return self.peers.pop().map(Event::Lost);
}
}
}
}
fn send(&mut self, to: &NodeId, frame: &[u8]) -> Result<(), SendError> {
if !self.allows(to) || !self.peers.contains(to) {
return Err(SendError::Unreachable);
}
if frame.len() > self.mtu() {
return Err(SendError::TooLarge);
}
match self.link.send(to, frame) {
Ok(()) => Ok(()),
// The only failure modes left are a dead link (nothing is
// reachable any more) and a bad destination.
Err(_) => Err(SendError::Unreachable),
}
}
fn mtu(&self) -> usize {
self.link.mtu()
}
}