boxmaker/crates/gatewayd/tests/serve_restart.rs

//! `gatewayd` end to end across gaps: a restart, a lost connection, an unreachable server, a
//! refused token and a damaged state file (M4a spec, section 9). Do not edit.

#[path = "support/fake_loop.rs"]
mod fake_loop;
#[path = "support/fake_mm.rs"]
mod fake_mm;
#[path = "support/gateway.rs"]
mod gateway;
#[path = "support/tmp.rs"]
mod tmp;

use std::sync::{Mutex, mpsc};
use std::time::Duration;

use fake_loop::{done, serve_loop};
use fake_mm::{DM, FakeMm, KYLE, id, post};
use gateway::{SHARED, WAIT, answering, config, loop_dir, start, up};
use gatewayd::serve::{INTERRUPTED, Stop};
use tmp::TempDir;

#[test]
fn a_restart_reports_the_cut_off_turn_and_catches_up() {
    let home = TempDir::new("serve-restart");
    let (cut, seen, new1, new2) = (id('r', 1), id('p', 2), id('p', 3), id('p', 4));
    let state = serde_json::json!({
        "channels": {DM: 1000}, "recent": [seen], "threads": [],
        "in_flight": [{"session": format!("mm-{cut}"), "channel": DM, "root": cut}]
    });
    home.write("gateway/state.json", &state.to_string());
    loop_dir(&home);
    let turns = answering(&home, Duration::ZERO);
    let mm = FakeMm::start();
    mm.set_since(
        DM,
        &[
            post(&new2, KYLE, DM, "", "second", 2000),
            post(&seen, KYLE, DM, "", "already answered", 1500),
            post(&new1, KYLE, DM, "", "first", 1800),
        ],
    );
    let _running = start(config(&home, &mm.url(), ""));
    let _ws = mm.next_ws(WAIT);
    // Two top-level posts are two sessions, whose turns run on their own threads: they may reach
    // `loopd` in either order. The order of catch-up itself is `mm_json.rs`'s to check.
    let mut got = [
        turns.recv_timeout(WAIT).unwrap().content,
        turns.recv_timeout(WAIT).unwrap().content,
    ];
    got.sort();
    assert_eq!(got, ["first", "second"]);
    let posts = mm.wait_posts(3, WAIT);
    assert_eq!(posts[0], (DM.to_string(), cut, INTERRUPTED.to_string()));
    assert_eq!(posts.len(), 3, "{posts:?}");
    std::thread::sleep(Duration::from_millis(100));
    let saved: serde_json::Value = serde_json::from_str(
        &std::fs::read_to_string(home.path().join("gateway/state.json")).unwrap(),
    )
    .unwrap();
    assert_eq!(saved["in_flight"], serde_json::json!([]));
    assert_eq!(saved["channels"][DM], 2000);
}

#[test]
fn a_first_start_answers_no_history() {
    let home = TempDir::new("serve-first");
    loop_dir(&home);
    let turns = answering(&home, Duration::ZERO);
    let mm = FakeMm::start();
    mm.set_since(DM, &[post(&id('p', 1), KYLE, DM, "", "old", 5)]);
    let running = start(config(&home, &mm.url(), ""));
    let _ws = mm.next_ws(WAIT);
    running.wait_log("connected to");
    assert!(turns.recv_timeout(Duration::from_millis(300)).is_err());
    assert!(
        !mm.calls().iter().any(|(_, p)| p.contains("since=")),
        "{:?}",
        mm.calls()
    );
    let saved: serde_json::Value = serde_json::from_str(
        &std::fs::read_to_string(home.path().join("gateway/state.json")).unwrap(),
    )
    .unwrap();
    assert!(
        saved["channels"][DM].as_i64().unwrap() > 1_700_000_000_000,
        "marked from now"
    );
    assert!(saved["channels"][SHARED].as_i64().is_some());
}

#[test]
fn a_lost_connection_is_made_again() {
    let home = TempDir::new("serve-reconnect");
    let (mm, running, turns, ws) = up(&home, Duration::ZERO);
    ws.drop_connection();
    let mut again = mm.next_ws(WAIT);
    let log = running.wait_log("lost the connection");
    assert!(
        log.iter()
            .any(|l| l.ends_with("see docs/runbook.md#mattermost-unreachable")),
        "{log:?}"
    );
    let p1 = id('p', 1);
    again.posted(&post(&p1, KYLE, DM, "", "still there?", 5), "D");
    assert_eq!(turns.recv_timeout(WAIT).unwrap().content, "still there?");
    assert_eq!(mm.wait_posts(1, WAIT).len(), 1);
}

#[test]
fn a_refused_token_stops_gatewayd() {
    let home = TempDir::new("serve-auth");
    let mm = FakeMm::start();
    mm.refuse_token();
    let running = start(config(&home, &mm.url(), ""));
    let stop = running.join_within();
    assert!(matches!(stop, Stop::Auth(401)), "{stop}");
    assert_eq!(
        stop.to_string(),
        "gatewayd: Mattermost refused the token (401)\nsee docs/runbook.md#mattermost-auth-failed"
    );
}

#[test]
fn an_unreachable_server_is_tried_again() {
    let home = TempDir::new("serve-unreachable");
    let port = std::net::TcpListener::bind("127.0.0.1:0")
        .unwrap()
        .local_addr()
        .unwrap()
        .port();
    let url = format!("http://127.0.0.1:{port}");
    let running = start(config(&home, &url, ""));
    std::thread::sleep(Duration::from_millis(300));
    let log = running.log();
    let tries: Vec<&String> = log
        .iter()
        .filter(|l| l.starts_with(&format!("gatewayd: cannot reach {url}: ")))
        .collect();
    assert!(tries.len() >= 2, "{log:?}");
    assert!(
        tries
            .iter()
            .all(|l| l
                .ends_with("; trying again in 0 s\nsee docs/runbook.md#mattermost-unreachable")),
        "{tries:?}"
    );
    assert!(matches!(running.finish(), Stop::Asked));
}

#[test]
fn a_damaged_state_file_stops_at_once() {
    let home = TempDir::new("serve-damaged");
    home.write("gateway/state.json", "{not json");
    let mm = FakeMm::start();
    let running = start(config(&home, &mm.url(), ""));
    let stop = running.join_within();
    assert!(matches!(stop, Stop::State(_)), "{stop}");
    assert!(
        stop.to_string()
            .ends_with("see docs/runbook.md#gateway-state-damaged"),
        "{stop}"
    );
    assert!(mm.calls().is_empty(), "nothing is asked of Mattermost");
}

#[test]
fn a_reconnect_does_not_interrupt_a_running_turn() {
    let home = TempDir::new("serve-reconnect-turn");
    loop_dir(&home);
    let (release_tx, release_rx) = mpsc::channel::<()>();
    let release = Mutex::new(release_rx);
    let turns = serve_loop(&home.path().join("run/loop/loop.sock"), move |_, _| {
        let _ = release.lock().unwrap().recv_timeout(WAIT);
        vec![done("the answer")]
    });
    let mm = FakeMm::start();
    let running = start(config(&home, &mm.url(), ""));
    let mut ws = mm.next_ws(WAIT);
    let p1 = id('p', 1);
    ws.posted(&post(&p1, KYLE, DM, "", "a long one", 5), "D");
    turns.recv_timeout(WAIT).unwrap();
    ws.drop_connection();
    let _again = mm.next_ws(WAIT);
    running.wait_log("lost the connection");
    std::thread::sleep(Duration::from_millis(100));
    release_tx.send(()).unwrap();
    let posts = mm.wait_posts(1, WAIT);
    std::thread::sleep(Duration::from_millis(200));
    assert_eq!(
        mm.posts(),
        [(DM.to_string(), p1, "the answer".to_string())],
        "{posts:?}"
    );
}