memoryd: the edit journal, undo of the latest edit, and review

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/memoryd/src/journal.rs

@@ -0,0 +1,232 @@
+//! The edit journal: one JSON line per applied edit in `memory/edits/YYYY-MM.jsonl`, the owner's
+//! record of every write, and `undo` of the latest edit of a file (M5a spec, section 5).
+
+use serde::{Deserialize, Serialize};
+
+use proto::{MemoryFile, Timestamp};
+
+use crate::edit::{Context, Outcome};
+use crate::files::Store;
+
+/// One line of the journal.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Record {
+    /// `YYYY-MM-DD-NNNN`, numbered from 1 each day.
+    pub id: String,
+    pub time: String,
+    pub session: String,
+    pub call: u64,
+    /// `add`, `replace`, `remove`, `consolidate`, `note`, or `undo`.
+    pub action: String,
+    /// The identity file changed, if any.
+    pub file: Option<String>,
+    /// Where the write went: `memory.md`, `user.md` or `notes.md`.
+    pub to: String,
+    pub before: Vec<String>,
+    pub after: Vec<String>,
+    /// The `notes.md` entries appended, as written.
+    pub notes: Vec<String>,
+    pub untrusted: bool,
+    /// For an undo, the edit it reverted.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub undoes: Option<String>,
+}
+
+fn file_of(name: &str) -> Option<MemoryFile> {
+    match name {
+        "memory.md" => Some(MemoryFile::Memory),
+        "user.md" => Some(MemoryFile::User),
+        _ => None,
+    }
+}
+
+/// Every record, oldest first, from every month's file.
+pub fn records(store: &Store) -> Result<Vec<Record>, String> {
+    // Read the directory `store.path("edits")`: NotFound -> Ok(empty); another error ->
+    // Err(format!("{}: {e}", dir.display())). Take the names that end ".jsonl" and sort them. For
+    // each file, `read_to_string` and parse every line as a Record with serde_json; a bad line ->
+    // Err(format!("{}: line {}: {e}", path.display(), n + 1)) with n counted from 0. Return all
+    // records, oldest first.
+    let dir = store.path("edits");
+    let mut paths: Vec<std::path::PathBuf> = match std::fs::read_dir(&dir) {
+        Ok(rd) => rd
+            .filter_map(|entry| entry.ok().map(|e| e.path()))
+            .collect(),
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
+        Err(e) => return Err(format!("{}: {e}", dir.display())),
+    };
+    paths.retain(|p| {
+        p.file_name()
+            .and_then(|s| s.to_str())
+            .is_some_and(|n| n.ends_with(".jsonl"))
+    });
+    paths.sort();
+    let mut out: Vec<Record> = Vec::new();
+    for path in paths {
+        let text =
+            std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
+        for (n, line) in text.lines().enumerate() {
+            let record: Record = serde_json::from_str(line)
+                .map_err(|e| format!("{}: line {}: {e}", path.display(), n + 1))?;
+            out.push(record);
+        }
+    }
+    Ok(out)
+}
+
+/// Append a record, giving it the next id of its day.
+pub(crate) fn append(store: &Store, mut record: Record, now: Timestamp) -> Result<String, String> {
+    // 1. `day` is the first 10 characters of `now.to_rfc3339()` ("0000-00-00" if shorter).
+    // 2. `taken` = how many of `records(store)?` have an id starting format!("{day}-").
+    // 3. record.id = format!("{day}-{:04}", taken + 1).
+    // 4. `create_dir_all(store.path("edits"))` (an error names the directory),
+    //    `serde_json::to_string(&record)`, then `store.append(&format!("edits/{month}.jsonl"),
+    //    &format!("{line}\n"))?` where month is the first 7 characters of day. Ok(record.id).
+    let rfc = now.to_rfc3339();
+    let day = if rfc.len() >= 10 {
+        &rfc[..10]
+    } else {
+        "0000-00-00"
+    };
+    let taken = records(store)?
+        .iter()
+        .filter(|r| r.id.starts_with(&format!("{day}-")))
+        .count();
+    record.id = format!("{day}-{:04}", taken + 1);
+    let edits_dir = store.path("edits");
+    std::fs::create_dir_all(&edits_dir).map_err(|e| format!("{}: {e}", edits_dir.display()))?;
+    let line = serde_json::to_string(&record).map_err(|e| e.to_string())?;
+    let month = &day[..7];
+    store.append(&format!("edits/{month}.jsonl"), &format!("{line}\n"))?;
+    Ok(record.id)
+}
+
+/// Journal an applied call; the id it was given.
+pub fn record(store: &Store, ctx: &Context, outcome: &Outcome) -> Result<String, String> {
+    // `append` a Record: id empty (append sets it), time `ctx.now.to_rfc3339()`, session and call
+    // from ctx, action `outcome.action`, file `change.file` as its name, to that name or "notes.md"
+    // when there is no file, before and after from the change, notes the change's notes each
+    // `render()`ed, untrusted `ctx.untrusted`, undoes None; `now` is ctx.now.
+    let change = &outcome.change;
+    let notes: Vec<String> = change.notes.iter().map(|n| n.render()).collect();
+    let file = change.file.map(|f| f.name().to_string());
+    let to = match &file {
+        Some(name) => name.clone(),
+        None => "notes.md".to_string(),
+    };
+    append(
+        store,
+        Record {
+            id: String::new(),
+            time: ctx.now.to_rfc3339(),
+            session: ctx.session.clone(),
+            call: ctx.call,
+            action: outcome.action.to_string(),
+            file,
+            to,
+            before: change.before.clone(),
+            after: change.after.clone(),
+            notes,
+            untrusted: ctx.untrusted,
+            undoes: None,
+        },
+        ctx.now,
+    )
+}
+
+/// The last `count` records, one line each, oldest first.
+pub fn log(store: &Store, count: usize) -> Result<String, String> {
+    // The last `count` of `records(store)?` (all of them when fewer), oldest first, one line each:
+    // format!("{}  {}  {}  {}  {}{}", id, time, session, action, to, and "  untrusted" when the
+    // record is untrusted), joined by newlines. With no records at all: "no memory edits yet".
+    let all = records(store)?;
+    if all.is_empty() {
+        return Ok("no memory edits yet".to_string());
+    }
+    let selected = &all[all.len().saturating_sub(count)..];
+    let lines: Vec<String> = selected
+        .iter()
+        .map(|r| {
+            if r.untrusted {
+                format!(
+                    "{}  {}  {}  {}  {}  untrusted",
+                    r.id, r.time, r.session, r.action, r.to
+                )
+            } else {
+                format!(
+                    "{}  {}  {}  {}  {}",
+                    r.id, r.time, r.session, r.action, r.to
+                )
+            }
+        })
+        .collect();
+    Ok(lines.join("\n"))
+}
+
+/// Revert edit `id`, if it is the latest edit of its identity file and the file still holds
+/// exactly what it left. Notes it appended stay (the file is append-only). The undo is journalled.
+pub fn undo(store: &Store, id: &str, now: Timestamp) -> Result<String, String> {
+    // 1. Find the record with this id in `records(store)?`, else Err(format!("no edit {id}")).
+    // 2. Its file must be Some and known to `file_of`, else Err(format!("{id} changed no identity
+    //    file; notes are append-only")).
+    // 3. The latest record whose file is the same name must be this one, else Err(format!("{id} is
+    //    not the latest edit of {name}")).
+    // 4. `store.entries(file)?` must equal record.after, else Err(format!("{name} has changed since
+    //    {id}; nothing was undone")).
+    // 5. `store.write_entries(file, &record.before)?`, then `append` a Record: session "owner",
+    //    call 0, action "undo", file and to the name, before the current entries, after
+    //    record.before, no notes, untrusted false, undoes Some(id).
+    // 6. Ok(format!("undone: {id} ({name} restored; journalled as {undo_id})")).
+    let all = records(store)?;
+    let record = all
+        .iter()
+        .find(|r| r.id == id)
+        .ok_or_else(|| format!("no edit {id}"))?;
+    let name: &str = match &record.file {
+        Some(n) => n,
+        None => {
+            return Err(format!(
+                "{id} changed no identity file; notes are append-only"
+            ));
+        }
+    };
+    let file = match file_of(name) {
+        Some(f) => f,
+        None => {
+            return Err(format!(
+                "{id} changed no identity file; notes are append-only"
+            ));
+        }
+    };
+    let latest = all.iter().rev().find(|r| r.file.as_deref() == Some(name));
+    if latest.is_none_or(|r| r.id != id) {
+        return Err(format!("{id} is not the latest edit of {name}"));
+    }
+    let current = store.entries(file)?;
+    if current != record.after {
+        return Err(format!("{name} has changed since {id}; nothing was undone"));
+    }
+    store.write_entries(file, &record.before)?;
+    let undo_id = append(
+        store,
+        Record {
+            id: String::new(),
+            time: now.to_rfc3339(),
+            session: "owner".to_string(),
+            call: 0,
+            action: "undo".to_string(),
+            file: Some(name.to_string()),
+            to: name.to_string(),
+            before: current,
+            after: record.before.clone(),
+            notes: Vec::new(),
+            untrusted: false,
+            undoes: Some(id.to_string()),
+        },
+        now,
+    )?;
+    Ok(format!(
+        "undone: {id} ({name} restored; journalled as {undo_id})"
+    ))
+}

crates/memoryd/src/lib.rs

@@ -5,5 +5,7 @@ pub mod config;
 pub mod edit;
 pub mod entries;
 pub mod files;
+pub mod journal;
 pub mod notes;
+pub mod review;
 pub mod socket;

crates/memoryd/src/review.rs

@@ -0,0 +1,252 @@
+//! Review (off by default, `[memory] review = true`): identity writes wait in
+//! `memory/pending.jsonl` for the owner's `accept` or `reject` instead of being applied (M5a spec,
+//! section 5). Notes are never staged.
+
+use serde::{Deserialize, Serialize};
+
+use proto::escape::escape_model_text;
+use proto::{DataClass, MemoryArgs, Timestamp};
+
+use crate::edit::{Context, apply};
+use crate::files::Store;
+use crate::journal::{self, Record};
+
+/// One line of `pending.jsonl`: a write waiting, or what became of one.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
+pub enum Line {
+    Waiting {
+        id: String,
+        time: String,
+        session: String,
+        call: u64,
+        taint: DataClass,
+        /// The call's arguments, as `brokerd` sent them.
+        arguments: String,
+    },
+    Accepted {
+        id: String,
+        time: String,
+        edit: String,
+    },
+    Rejected {
+        id: String,
+        time: String,
+    },
+}
+
+fn lines(store: &Store) -> Result<Vec<Line>, String> {
+    // Read `store.path("pending.jsonl")`: NotFound -> Ok(empty); another error -> Err(format!("{}:
+    // {e}", path.display())). Parse every line as a Line with serde_json; a bad one ->
+    // Err(format!("{}: line {}: {e}", path.display(), n + 1)).
+    let path = store.path("pending.jsonl");
+    let text = match std::fs::read_to_string(&path) {
+        Ok(t) => t,
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
+        Err(e) => return Err(format!("{}: {e}", path.display())),
+    };
+    let mut out: Vec<Line> = Vec::new();
+    for (n, line) in text.lines().enumerate() {
+        let entry: Line = serde_json::from_str(line)
+            .map_err(|e| format!("{}: line {}: {e}", path.display(), n + 1))?;
+        out.push(entry);
+    }
+    Ok(out)
+}
+
+fn add(store: &Store, line: &Line) -> Result<(), String> {
+    // `serde_json::to_string(line)` (its error as a String), then `store.append("pending.jsonl",
+    // &format!("{text}\n"))`.
+    let text = serde_json::to_string(line).map_err(|e| e.to_string())?;
+    store.append("pending.jsonl", &format!("{text}\n"))
+}
+
+/// The writes still waiting: `(id, session, arguments, taint, call)`.
+fn waiting(store: &Store) -> Result<Vec<Line>, String> {
+    // The ids that have an Accepted or a Rejected line are done. Return a clone of every Waiting
+    // line whose id is not done, in file order.
+    let all = lines(store)?;
+    let done: Vec<String> = all
+        .iter()
+        .filter_map(|line| match line {
+            Line::Accepted { id, .. } | Line::Rejected { id, .. } => Some(id.clone()),
+            Line::Waiting { .. } => None,
+        })
+        .collect();
+    Ok(all
+        .iter()
+        .filter(|line| match line {
+            Line::Waiting { id, .. } => !done.iter().any(|d| d == id),
+            Line::Accepted { .. } | Line::Rejected { .. } => false,
+        })
+        .cloned()
+        .collect())
+}
+
+/// Stage an identity write. Its shape is checked now; its content when it is accepted.
+pub fn submit(
+    store: &Store,
+    args: &MemoryArgs,
+    arguments: &str,
+    ctx: &Context,
+) -> Result<String, String> {
+    // 1. `args.check()?`.
+    // 2. `day` is the first 10 characters of `ctx.now.to_rfc3339()`. `taken` = how many Waiting
+    //    lines have an id starting format!("p-{day}-"). The id is format!("p-{day}-{:04}", taken +
+    //    1).
+    // 3. `add` a Waiting line: id, time `ctx.now.to_rfc3339()`, session, call and taint from ctx,
+    //    arguments as given.
+    // 4. Ok(format!("waiting for the owner's review ({id}); nothing is changed until it is
+    //    accepted")).
+    args.check()?;
+    let rfc = ctx.now.to_rfc3339();
+    let day = &rfc[..10];
+    let taken = waiting(store)?
+        .iter()
+        .filter(|line| match line {
+            Line::Waiting { id, .. } => id.starts_with(&format!("p-{day}-")),
+            _ => false,
+        })
+        .count();
+    let id = format!("p-{day}-{:04}", taken + 1);
+    add(
+        store,
+        &Line::Waiting {
+            id: id.clone(),
+            time: rfc,
+            session: ctx.session.clone(),
+            call: ctx.call,
+            taint: ctx.taint,
+            arguments: arguments.to_string(),
+        },
+    )?;
+    Ok(format!(
+        "waiting for the owner's review ({id}); nothing is changed until it is accepted"
+    ))
+}
+
+/// The writes waiting, one per line, arguments escaped.
+pub fn list(store: &Store) -> Result<String, String> {
+    // One line per `waiting` entry: format!("{id}  {session}  {}", escape_model_text(arguments)),
+    // joined by newlines; with none, "nothing is waiting for review".
+    let waits = waiting(store)?;
+    let lines: Vec<String> = waits
+        .iter()
+        .filter_map(|l| match l {
+            Line::Waiting {
+                id,
+                session,
+                arguments,
+                ..
+            } => Some(format!("{id}  {session}  {}", escape_model_text(arguments))),
+            Line::Accepted { .. } | Line::Rejected { .. } => None,
+        })
+        .collect();
+    if lines.is_empty() {
+        Ok("nothing is waiting for review".to_string())
+    } else {
+        Ok(lines.join("\n"))
+    }
+}
+
+/// Apply a waiting write, with the same checks, against the file as it is now, and journal it.
+pub fn accept(store: &Store, id: &str, now: Timestamp) -> Result<String, String> {
+    // 1. The Waiting line with this id among `waiting(store)?`, else Err(format!("nothing waiting
+    //    as {id}")).
+    // 2. Parse its arguments as MemoryArgs (the error as a String).
+    // 3. A Context from its session, call and taint, untrusted false, now.
+    // 4. `apply(store, &args, &ctx)?`, then `journal::record(store, &ctx, &outcome)?` gives the
+    //    edit id.
+    // 5. `add` an Accepted line { id, time now, edit }.
+    // 6. Ok(format!("accepted: {id} as {edit}: {}", outcome.answer)).
+    let waiting_lines = waiting(store)?;
+    let found = waiting_lines
+        .iter()
+        .find(|l| match l {
+            Line::Waiting { id: l_id, .. } => l_id == id,
+            Line::Accepted { .. } | Line::Rejected { .. } => false,
+        })
+        .ok_or_else(|| format!("nothing waiting as {id}"))?;
+    let (args, session, call, taint) = match found {
+        Line::Waiting {
+            arguments,
+            session,
+            call,
+            taint,
+            ..
+        } => {
+            let args: MemoryArgs = serde_json::from_str(arguments).map_err(|e| e.to_string())?;
+            (args, session.clone(), *call, *taint)
+        }
+        Line::Accepted { .. } | Line::Rejected { .. } => {
+            return Err(format!("nothing waiting as {id}"));
+        }
+    };
+    let ctx = Context {
+        session,
+        call,
+        taint,
+        untrusted: false,
+        now,
+    };
+    let outcome = apply(store, &args, &ctx)?;
+    let edit = journal::record(store, &ctx, &outcome)?;
+    add(
+        store,
+        &Line::Accepted {
+            id: id.to_string(),
+            time: now.to_rfc3339(),
+            edit: edit.clone(),
+        },
+    )?;
+    Ok(format!("accepted: {id} as {edit}: {}", outcome.answer))
+}
+
+/// Discard a waiting write; the journal records it.
+pub fn reject(store: &Store, id: &str, now: Timestamp) -> Result<String, String> {
+    // 1. The Waiting line with this id, else Err(format!("nothing waiting as {id}")).
+    // 2. `add` a Rejected line { id, time now }.
+    // 3. `journal::append` a Record: its session and call, action "reject", file None, to empty, no
+    //    entries, no notes, untrusted false, undoes Some(id).
+    // 4. Ok(format!("rejected: {id}")).
+    let waiting_lines = waiting(store)?;
+    let found = waiting_lines
+        .iter()
+        .find(|l| match l {
+            Line::Waiting { id: l_id, .. } => l_id == id,
+            Line::Accepted { .. } | Line::Rejected { .. } => false,
+        })
+        .ok_or_else(|| format!("nothing waiting as {id}"))?;
+    let (session, call) = match found {
+        Line::Waiting { session, call, .. } => (session.clone(), *call),
+        Line::Accepted { .. } | Line::Rejected { .. } => {
+            return Err(format!("nothing waiting as {id}"));
+        }
+    };
+    add(
+        store,
+        &Line::Rejected {
+            id: id.to_string(),
+            time: now.to_rfc3339(),
+        },
+    )?;
+    journal::append(
+        store,
+        Record {
+            id: String::new(),
+            time: now.to_rfc3339(),
+            session,
+            call,
+            action: "reject".to_string(),
+            file: None,
+            to: String::new(),
+            before: Vec::new(),
+            after: Vec::new(),
+            notes: Vec::new(),
+            untrusted: false,
+            undoes: Some(id.to_string()),
+        },
+        now,
+    )?;
+    Ok(format!("rejected: {id}"))
+}