brokerd: courier, carrying pending approvals to gatewayd and answers back

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/brokerd/src/broker.rs

@@ -6,7 +6,7 @@
 use std::io::Read;
 use std::os::unix::net::UnixStream;
 use std::sync::Mutex;
-use std::sync::mpsc::RecvTimeoutError;
+use std::sync::mpsc::{Receiver, RecvTimeoutError};
 use std::time::Duration;
 
 use proto::{
@@ -16,6 +16,7 @@ use proto::{
 
 use crate::approvals::{Table, Verdict};
 use crate::config::Config;
+use crate::courier;
 use crate::grants;
 use crate::ledger::{Call, Decided, Grants, Ledger};
 use crate::policy::{Ask, SessionState};
@@ -166,6 +167,44 @@ pub fn forbid(broker: &Broker, stream: &mut UnixStream, id: u64, msg: &Message,
     );
 }
 
+/// What waiting for the verdict came to.
+enum Waited {
+    Verdict(Result<Verdict, DenyReason>),
+    PeerLeft,
+}
+
+/// Wait for the verdict, or for the peer to leave. If the peer leaves and the entry is still in
+/// the table, taking it ends the approval unanswered; if someone already took it, their verdict
+/// is awaited.
+fn wait_for_verdict(
+    broker: &Broker,
+    stream: &UnixStream,
+    verdict: &Receiver<Verdict>,
+    seq: u64,
+) -> Waited {
+    loop {
+        match verdict.recv_timeout(Duration::from_secs(1)) {
+            Ok(verdict) => return Waited::Verdict(Ok(verdict)),
+            Err(RecvTimeoutError::Timeout) => {
+                if alive(stream) {
+                    continue;
+                }
+                match broker.table().take(seq) {
+                    Some(_) => return Waited::PeerLeft,
+                    None => match verdict.recv() {
+                        Ok(verdict) => return Waited::Verdict(Ok(verdict)),
+                        Err(_) => return Waited::Verdict(Err(DenyReason::AuditUnavailable)),
+                    },
+                }
+            }
+            // The taker dropped it unanswered.
+            Err(RecvTimeoutError::Disconnected) => {
+                return Waited::Verdict(Err(DenyReason::AuditUnavailable));
+            }
+        }
+    }
+}
+
 /// Whether the peer is still there: it sends nothing more and never half-closes, so a byte would
 /// break the protocol and a timeout means it is waiting.
 pub fn alive(stream: &UnixStream) -> bool {
@@ -256,6 +295,7 @@ fn pending(
         created: now,
         expires,
     };
+    let shown = info.clone();
     let verdict = broker.table().insert(info, ask);
 
     // 3. Show the caller the pending call. If the frame does not reach the peer, the entry is
@@ -273,24 +313,14 @@ fn pending(
         return None; // 3. the peer left before the frame went out
     }
 
-    // 4. Wait for the verdict, or for the peer to leave.
-    let outcome = loop {
-        match verdict.recv_timeout(Duration::from_secs(1)) {
-            Ok(verdict) => break Ok(verdict),
-            Err(RecvTimeoutError::Timeout) => {
-                if alive(stream) {
-                    continue;
-                }
-                match broker.table().take(seq) {
-                    Some(_) => return None, // 4. the peer left while we waited
-                    None => match verdict.recv() {
-                        Ok(verdict) => break Ok(verdict),
-                        Err(_) => break Err(DenyReason::AuditUnavailable),
-                    },
-                }
-            }
-            Err(RecvTimeoutError::Disconnected) => break Err(DenyReason::AuditUnavailable), // 4. the taker dropped it unanswered
-        }
+    // 4. Wait for the verdict, or for the peer to leave, with a courier carrying the approval to
+    //    Mattermost meanwhile when `[gateway]` is set.
+    let waited = courier::alongside(broker, &shown, || {
+        wait_for_verdict(broker, stream, &verdict, seq)
+    });
+    let outcome = match waited {
+        Waited::Verdict(outcome) => outcome,
+        Waited::PeerLeft => return None, // 4. the peer left while we waited
     };
 
     // 5. Answer from the verdict.