feat(store): quarantine held observations until a wall-clock downtime fold
This commit is contained in:
@@ -486,3 +486,517 @@ mod fleet_fixes_f5 {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// The quarantine + downtime-fold tests (Task 10). They reopen logs that were
|
||||
// written while "down" and inspect the held/quarantined state before and after
|
||||
// the host trusts a wall clock. The marker is established through the public
|
||||
// API (set_wall_clock + flush) so nothing here reaches into crate internals.
|
||||
mod quarantine_and_downtime {
|
||||
use super::*;
|
||||
use tines_core::Store;
|
||||
use tines_store::LogStore;
|
||||
|
||||
fn addressed(observer: [u8; 16], seq: u64, to: [u8; 16], lifetime: u64) -> Observation {
|
||||
Observation::addressed(observer, seq, b"task", b"go", to, lifetime)
|
||||
}
|
||||
|
||||
/// Held (quarantined) observations are withheld from every delivery pass
|
||||
/// until the host trusts a wall clock.
|
||||
#[test]
|
||||
fn held_is_quarantined_until_wall_clock() {
|
||||
let path = store_path("held_quarantined");
|
||||
// Seed a broadcast and an addressed observation, then trust a wall
|
||||
// clock so the marker below carries a wall (a *known* downtime on
|
||||
// reopen) rather than a bare one.
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).expect("open fresh");
|
||||
let b = obs([1; 16], 1, 0, Some(0));
|
||||
let a = addressed([2; 16], 1, [3; 16], 60_000).with_observed_at(0);
|
||||
s.insert(b, 0).unwrap();
|
||||
s.insert(a, 0).unwrap();
|
||||
s.set_wall_clock(500, 0);
|
||||
s.flush(0).unwrap();
|
||||
}
|
||||
|
||||
// Reopen after a downtime. Until set_wall_clock the copies are held.
|
||||
{
|
||||
let mut s = LogStore::open(&path, 1000).expect("reopen");
|
||||
let b_id = obs([1; 16], 1, 0, Some(0)).id();
|
||||
let a_id = addressed([2; 16], 1, [3; 16], 60_000).id();
|
||||
|
||||
assert!(s.contains(&b_id), "broadcast still present while held");
|
||||
assert!(s.contains(&a_id), "addressed still present while held");
|
||||
assert!(
|
||||
s.is_released(&a_id),
|
||||
"a held addressed observation is treated as released"
|
||||
);
|
||||
|
||||
let mut broadcasts = 0usize;
|
||||
s.for_each_broadcast_id(&mut |_| broadcasts += 1);
|
||||
assert_eq!(broadcasts, 0, "held broadcasts are not delivered");
|
||||
let mut addressed = 0usize;
|
||||
s.for_each_addressed_id(&mut |_| addressed += 1);
|
||||
assert_eq!(
|
||||
addressed, 0,
|
||||
"held addressed observations are not delivered"
|
||||
);
|
||||
let mut topics = 0usize;
|
||||
s.for_each_topic(b"", 1000, &mut |_obs| {
|
||||
topics += 1;
|
||||
});
|
||||
assert_eq!(topics, 0, "held observations are not offered for exchange");
|
||||
|
||||
// Trust the wall clock: the downtime folds and the quarantine lifts.
|
||||
let report = s.set_wall_clock(1000, 1000);
|
||||
assert_eq!(
|
||||
report.downtime_ms,
|
||||
Some(500),
|
||||
"known downtime since the marker"
|
||||
);
|
||||
|
||||
let mut broadcasts = 0usize;
|
||||
s.for_each_broadcast_id(&mut |_| broadcasts += 1);
|
||||
assert_eq!(
|
||||
broadcasts, 1,
|
||||
"the broadcast is delivered again after trust"
|
||||
);
|
||||
let mut addressed = 0usize;
|
||||
s.for_each_addressed_id(&mut |_| addressed += 1);
|
||||
assert_eq!(addressed, 1, "the addressed observation is delivered again");
|
||||
let mut topics = 0usize;
|
||||
s.for_each_topic(b"", 1000, &mut |_obs| {
|
||||
topics += 1;
|
||||
});
|
||||
assert!(topics >= 1, "topics are visited after trust");
|
||||
}
|
||||
}
|
||||
|
||||
/// An observation inserted into a freshly opened store is never held.
|
||||
#[test]
|
||||
fn new_inserts_not_quarantined() {
|
||||
let path = store_path("fresh_insert");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let b = obs([1; 16], 1, 0, Some(0));
|
||||
let a = addressed([2; 16], 1, [3; 16], 60_000).with_observed_at(0);
|
||||
let a_id = a.id();
|
||||
assert!(s.insert(b.clone(), 0).unwrap());
|
||||
assert!(s.insert(a.clone(), 0).unwrap());
|
||||
|
||||
// Delivered at once: nothing was held across a downtime.
|
||||
let mut broadcasts = 0usize;
|
||||
s.for_each_broadcast_id(&mut |_| broadcasts += 1);
|
||||
assert_eq!(broadcasts, 1);
|
||||
let mut addressed = 0usize;
|
||||
s.for_each_addressed_id(&mut |_| addressed += 1);
|
||||
assert_eq!(addressed, 1);
|
||||
|
||||
// The fold cannot touch a held observation that was never held.
|
||||
let before_b = s.get(&b.id(), 0).unwrap().age_ms;
|
||||
let before_a = s.get(&a_id, 0).unwrap().age_ms;
|
||||
s.set_wall_clock(2000, 0);
|
||||
assert_eq!(
|
||||
s.get(&b.id(), 0).unwrap().age_ms,
|
||||
before_b,
|
||||
"the fold leaves freshly-inserted broadcasts alone"
|
||||
);
|
||||
assert_eq!(
|
||||
s.get(&a_id, 0).unwrap().age_ms,
|
||||
before_a,
|
||||
"the fold leaves freshly-inserted addressed observations alone"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// With a known downtime, set_wall_clock folds the wall-clock elapsed into
|
||||
/// every held age.
|
||||
#[test]
|
||||
fn downtime_known_advances_ages() {
|
||||
let path = store_path("known_downtime");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let b = obs([1; 16], 1, 0, Some(0));
|
||||
s.insert(b, 0).unwrap();
|
||||
// Trust a wall clock at 1000 so the marker below carries a wall.
|
||||
s.set_wall_clock(1000, 0);
|
||||
s.flush(0).unwrap();
|
||||
}
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let before = s.get(&obs([1; 16], 1, 0, Some(0)).id(), 0).unwrap().age_ms;
|
||||
assert_eq!(before, 0);
|
||||
|
||||
// The caller clock is frozen at 0 while the wall climbs 1000 -> 2000.
|
||||
let report = s.set_wall_clock(2000, 0);
|
||||
assert_eq!(report.downtime_ms, Some(1000));
|
||||
let after = s.get(&obs([1; 16], 1, 0, Some(0)).id(), 0).unwrap().age_ms;
|
||||
// Held age advanced by the wall downtime (2000 - 1000 = 1000),
|
||||
// minus the ticks already counted during reopen (0).
|
||||
assert_eq!(after, 1000);
|
||||
}
|
||||
}
|
||||
|
||||
/// Without a marker, the downtime is unknown and held addressed observations
|
||||
/// expire (and are counted as such).
|
||||
#[test]
|
||||
fn downtime_unknown_expires_addressed() {
|
||||
let path = store_path("unknown_expires");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let a = addressed([2; 16], 1, [3; 16], 10_000).with_observed_at(0);
|
||||
assert!(s.insert(a, 0).unwrap());
|
||||
// Drop without flushing: no Clock marker on disk.
|
||||
drop(s);
|
||||
}
|
||||
{
|
||||
let mut s = LogStore::open(&path, 2000).unwrap();
|
||||
let mut delivered = 0usize;
|
||||
s.for_each_addressed_id(&mut |_| delivered += 1);
|
||||
assert_eq!(
|
||||
delivered, 0,
|
||||
"a held addressed observation is never delivered"
|
||||
);
|
||||
|
||||
let report = s.set_wall_clock(2000, 2000);
|
||||
assert_eq!(
|
||||
report.downtime_ms, None,
|
||||
"no marker on disk -> unknown downtime"
|
||||
);
|
||||
assert_eq!(
|
||||
report.expired_addressed, 1,
|
||||
"the held addressed observation is aged past its lifetime"
|
||||
);
|
||||
let mut delivered = 0usize;
|
||||
s.for_each_addressed_id(&mut |_| delivered += 1);
|
||||
assert_eq!(
|
||||
delivered, 0,
|
||||
"an expired held observation is never delivered"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A marker without a wall time is treated as unknown downtime, so held
|
||||
/// addressed observations expire.
|
||||
#[test]
|
||||
fn marker_without_wall_is_unknown() {
|
||||
let path = store_path("marker_no_wall");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let a = addressed([2; 16], 1, [3; 16], 10_000).with_observed_at(0);
|
||||
assert!(s.insert(a, 0).unwrap());
|
||||
// Flush BEFORE set_wall_clock: the marker carries wall:None.
|
||||
s.flush(0).unwrap();
|
||||
}
|
||||
{
|
||||
let mut s = LogStore::open(&path, 2000).unwrap();
|
||||
let report = s.set_wall_clock(2000, 2000);
|
||||
assert_eq!(
|
||||
report.downtime_ms, None,
|
||||
"a marker without a wall is unknown"
|
||||
);
|
||||
assert_eq!(report.expired_addressed, 1);
|
||||
let mut delivered = 0usize;
|
||||
s.for_each_addressed_id(&mut |_| delivered += 1);
|
||||
assert_eq!(delivered, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Without a known downtime, a held broadcast is floored to the configured
|
||||
/// unknown-downtime age floor.
|
||||
#[test]
|
||||
fn downtime_unknown_floors_broadcasts() {
|
||||
let path = store_path("floor_broadcasts");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let b = obs([1; 16], 1, 0, Some(0));
|
||||
assert!(s.insert(b, 0).unwrap());
|
||||
s.flush(0).unwrap();
|
||||
}
|
||||
{
|
||||
// Configure the floor explicitly via open_with (the builder's
|
||||
// setter is unexposed; open_with reaches the same field).
|
||||
let mut s = LogStore::open_with(&path, 2000, 30_000).unwrap();
|
||||
let report = s.set_wall_clock(2000, 2000);
|
||||
assert_eq!(report.downtime_ms, None);
|
||||
assert_eq!(report.floored_broadcasts, 1, "a held broadcast is floored");
|
||||
let g = s.get(&obs([1; 16], 1, 0, Some(0)).id(), 2000).unwrap();
|
||||
assert_eq!(
|
||||
g.age_ms, 30_000,
|
||||
"the held broadcast's age is raised to unknown_downtime_age_ms"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A marker whose wall time is ahead of now means the clock went backwards;
|
||||
/// the downtime is unknown.
|
||||
#[test]
|
||||
fn clock_went_backwards_is_unknown() {
|
||||
let path = store_path("clock_backwards");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let a = addressed([2; 16], 1, [3; 16], 10_000).with_observed_at(0);
|
||||
assert!(s.insert(a, 0).unwrap());
|
||||
// Trust the wall at 5000 before the marker is written.
|
||||
s.set_wall_clock(5000, 0);
|
||||
s.flush(0).unwrap();
|
||||
}
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let report = s.set_wall_clock(1000, 0);
|
||||
assert_eq!(
|
||||
report.downtime_ms, None,
|
||||
"a wall behind the marker (clock went backwards) is unknown"
|
||||
);
|
||||
assert_eq!(report.expired_addressed, 1);
|
||||
let mut delivered = 0usize;
|
||||
s.for_each_addressed_id(&mut |_| delivered += 1);
|
||||
assert_eq!(delivered, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// The wall a marker carries is derived: flush() after set_wall_clock(W, T)
|
||||
/// writes wall W + (flush_now - T); a flush before set_wall_clock writes
|
||||
/// wall:None.
|
||||
#[test]
|
||||
fn marker_wall_time_is_derived() {
|
||||
// (a) A flush before set_wall_clock writes a marker with wall:None.
|
||||
{
|
||||
let path = store_path("marker_derived_none");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let b = obs([1; 16], 1, 0, Some(0));
|
||||
s.insert(b, 0).unwrap();
|
||||
s.flush(0).unwrap(); // no wall clock yet
|
||||
}
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let report = s.set_wall_clock(15_000, 0);
|
||||
assert_eq!(
|
||||
report.downtime_ms, None,
|
||||
"a flush before set_wall_clock writes wall:None"
|
||||
);
|
||||
}
|
||||
}
|
||||
// (b) set_wall_clock(W, T) then flush(T + 5000) writes W + 5000.
|
||||
{
|
||||
let path = store_path("marker_derived_wall");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let b = obs([1; 16], 1, 0, Some(0));
|
||||
s.insert(b, 0).unwrap();
|
||||
s.set_wall_clock(5000, 1000); // wall_at_set = 5000
|
||||
s.flush(6000).unwrap(); // marker wall = 5000 + (6000 - 1000)
|
||||
}
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
// Known downtime = wall_now(15000) - marker_wall(10000) = 5000,
|
||||
// which only holds if the marker carried the derived wall.
|
||||
let report = s.set_wall_clock(15_000, 0);
|
||||
assert_eq!(report.downtime_ms, Some(5000));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// set_wall_clock runs exactly once: a second call is a no-op.
|
||||
#[test]
|
||||
fn set_wall_clock_runs_once() {
|
||||
let path = store_path("set_once");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let b = obs([1; 16], 1, 0, Some(0));
|
||||
s.insert(b, 0).unwrap();
|
||||
s.flush(0).unwrap();
|
||||
}
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let r1 = s.set_wall_clock(2000, 0);
|
||||
let before = s.get(&obs([1; 16], 1, 0, Some(0)).id(), 0).unwrap().age_ms;
|
||||
let r2 = s.set_wall_clock(2000, 0);
|
||||
let after = s.get(&obs([1; 16], 1, 0, Some(0)).id(), 0).unwrap().age_ms;
|
||||
assert_eq!(r2, r1, "a second set_wall_clock returns the first report");
|
||||
assert_eq!(before, after, "a second set_wall_clock changes nothing");
|
||||
assert_eq!(r2.downtime_ms, None, "the fold still ran once");
|
||||
}
|
||||
}
|
||||
|
||||
/// The FoldReport is readable after the fold (like skipped_records).
|
||||
#[test]
|
||||
fn fold_reported_once() {
|
||||
let path = store_path("fold_report");
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
let b = obs([1; 16], 1, 0, Some(0));
|
||||
let a = addressed([2; 16], 1, [3; 16], 1_000).with_observed_at(0);
|
||||
assert!(s.insert(b, 0).unwrap());
|
||||
assert!(s.insert(a, 0).unwrap());
|
||||
s.flush(0).unwrap();
|
||||
}
|
||||
{
|
||||
let mut s = LogStore::open(&path, 0).unwrap();
|
||||
assert!(s.fold_report().is_none(), "no fold has happened yet");
|
||||
|
||||
let r1 = s.set_wall_clock(2000, 0);
|
||||
let report = s
|
||||
.fold_report()
|
||||
.expect("the FoldReport is readable after the fold");
|
||||
assert_eq!(r1, report);
|
||||
assert_eq!(
|
||||
report.downtime_ms, None,
|
||||
"no marker on disk -> unknown downtime"
|
||||
);
|
||||
assert_eq!(report.floored_broadcasts, 1);
|
||||
|
||||
// It ran once: the report is unchanged by a second call.
|
||||
let r2 = s.set_wall_clock(2000, 0);
|
||||
assert_eq!(r2, r1, "a second call returns the report unchanged");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An exchange-level test over LogStore + a mock transport. The store is
|
||||
// LogStore (tines-sim's harness uses MemStore), so the exchange is driven
|
||||
// directly through Node<LogStore, Mock>.
|
||||
mod quarantined_duplicate_exchange {
|
||||
use super::*;
|
||||
use std::cell::{Cell, RefCell};
|
||||
use std::collections::VecDeque;
|
||||
use std::rc::Rc;
|
||||
use tines_core::{consts, Event, Node, NodeId, SendError, Store, Transport};
|
||||
use tines_store::LogStore;
|
||||
|
||||
fn addressed(observer: [u8; 16], seq: u64, to: [u8; 16], lifetime: u64) -> Observation {
|
||||
Observation::addressed(observer, seq, b"task", b"go", to, lifetime)
|
||||
}
|
||||
|
||||
/// A point-to-point link: a send on one side is delivered verbatim as a
|
||||
/// frame on the other. `sent` counts every frame delivered over it.
|
||||
struct Link {
|
||||
rx_a: RefCell<VecDeque<Event<NodeId>>>,
|
||||
rx_b: RefCell<VecDeque<Event<NodeId>>>,
|
||||
sent: Cell<usize>,
|
||||
}
|
||||
|
||||
struct Mock {
|
||||
own: NodeId,
|
||||
link: Rc<RefCell<Link>>,
|
||||
}
|
||||
|
||||
impl Transport for Mock {
|
||||
type Addr = NodeId;
|
||||
|
||||
fn poll(&mut self) -> Option<Event<NodeId>> {
|
||||
match self.own[0] {
|
||||
1 => self.link.borrow_mut().rx_a.borrow_mut().pop_front(),
|
||||
2 => self.link.borrow_mut().rx_b.borrow_mut().pop_front(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn send(&mut self, to: &NodeId, frame: &[u8]) -> Result<(), SendError> {
|
||||
let to = *to;
|
||||
let buf = tines_core::FrameBuf::from_slice(frame).expect("frame fits MAX_FRAME");
|
||||
let link = self.link.borrow_mut();
|
||||
match (self.own[0], to[0]) {
|
||||
(1, 2) => link.rx_b.borrow_mut().push_back(Event::Frame(to, buf)),
|
||||
(2, 1) => link.rx_a.borrow_mut().push_back(Event::Frame(to, buf)),
|
||||
_ => return Err(SendError::Unreachable),
|
||||
}
|
||||
link.sent.set(link.sent.get() + 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mtu(&self) -> usize {
|
||||
consts::MAX_FRAME
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quarantined_addressed_duplicate_not_acked() {
|
||||
use std::cell::Cell;
|
||||
use std::collections::VecDeque;
|
||||
use std::rc::Rc;
|
||||
|
||||
let a_id = [1u8; 16];
|
||||
let b_id = [2u8; 16];
|
||||
let id = |s: u64| addressed(a_id, s, b_id, 60_000).id();
|
||||
|
||||
let a_path = store_path("quar_a");
|
||||
let b_path = store_path("quar_b");
|
||||
|
||||
// A holds the addressed observation, inserted into a store that is used
|
||||
// directly by its node (never reopened), so it is never held.
|
||||
let mut a_store = LogStore::open(&a_path, 0).unwrap();
|
||||
assert!(a_store.insert(addressed(a_id, 0, b_id, 60_000), 0).unwrap());
|
||||
|
||||
// B holds the *same* observation, but quarantined across a downtime: a
|
||||
// wall-bearing marker (set via the public API) + a reopen after the wall.
|
||||
{
|
||||
let mut b = LogStore::open(&b_path, 0).unwrap();
|
||||
assert!(b.insert(addressed(a_id, 0, b_id, 60_000), 0).unwrap());
|
||||
b.set_wall_clock(1000, 0);
|
||||
b.flush(0).unwrap();
|
||||
}
|
||||
let b_store = LogStore::open(&b_path, 2000).unwrap();
|
||||
assert!(
|
||||
b_store.is_released(&id(0)),
|
||||
"B's copy is quarantined before the exchange"
|
||||
);
|
||||
|
||||
let link = Rc::new(RefCell::new(Link {
|
||||
rx_a: RefCell::new(VecDeque::new()),
|
||||
rx_b: RefCell::new(VecDeque::new()),
|
||||
sent: Cell::new(0),
|
||||
}));
|
||||
let mut a = Node::new(
|
||||
a_id,
|
||||
0,
|
||||
a_store,
|
||||
Mock {
|
||||
own: a_id,
|
||||
link: link.clone(),
|
||||
},
|
||||
);
|
||||
let mut b = Node::new(
|
||||
b_id,
|
||||
0,
|
||||
b_store,
|
||||
Mock {
|
||||
own: b_id,
|
||||
link: link.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
// Establish the link: A hears about B, B hears about A.
|
||||
link.borrow_mut()
|
||||
.rx_a
|
||||
.borrow_mut()
|
||||
.push_back(Event::Contact(b_id));
|
||||
link.borrow_mut()
|
||||
.rx_b
|
||||
.borrow_mut()
|
||||
.push_back(Event::Contact(a_id));
|
||||
|
||||
// Drive the exchange until no more frames are produced.
|
||||
loop {
|
||||
let before = link.borrow().sent.get();
|
||||
a.poll(2000);
|
||||
b.poll(2000);
|
||||
if link.borrow().sent.get() == before {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// B held a duplicate of a quarantined addressed observation, so it must
|
||||
// NOT ack. A never releasing custody proves no Ack came back: a released
|
||||
// duplicate would have acked and released A.
|
||||
assert!(
|
||||
!a.store().is_released(&id(0)),
|
||||
"a quarantined duplicate must not ack — A's custody stays held"
|
||||
);
|
||||
// Non-vacuous: the exchange ran and A offered the held observation.
|
||||
assert!(
|
||||
link.borrow().sent.get() >= 1,
|
||||
"A must have offered the held observation to B"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user