bxcore: sanitize::refuse, what a memory write may never hold

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>

crates/bxcore/src/sanitize/mod.rs

@@ -1,6 +1,9 @@
 //! The sanitizer (M5b spec, section 6): what is done to text from outside before it reaches the
-//! model or memory. Neutralising the model's chat-template tags (6.2) is in `tags`.
+//! model or memory. Neutralising the model's chat-template tags (6.2) is in `tags`; refusing what a
+//! memory write may never hold (6.1) is in `refuse`.
 
+mod refuse;
 mod tags;
 
+pub use refuse::{Refusal, refuse};
 pub use tags::{TAGS, Tag, neutralise};

crates/bxcore/src/sanitize/refuse.rs

@@ -0,0 +1,367 @@
+//! Refuse (M5b spec, section 6.1): what a memory write may never hold. Each rule matches bytes or
+//! strings that are only ever an attack or a leaked secret, never ordinary code or notes about
+//! code: hex and base64 runs, links, fences, commands and paths all pass (the code-work corpus in
+//! `tests/sanitize_refuse_corpus.rs`).
+//!
+//! The control, invisible and tag rules are M5a's (`memoryd`'s `check.rs`). The tag rule is
+//! narrower: it matches the listed tags exactly, as `neutralise` does, where M5a matched pieces of
+//! them in any case. M5a's phrase rule is not a refusal here but a flag (6.3).
+
+use super::tags::next_tag;
+
+/// The first rule a text breaks.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Refusal {
+    /// The rule's stable name, as the journal and `bxctl sanitize check` show it.
+    pub rule: &'static str,
+    /// The sentence memory answers with. Fixed: it never quotes the text.
+    pub sentence: &'static str,
+    /// The byte offset in the text where the rule matched.
+    pub at: usize,
+}
+
+struct Rule {
+    name: &'static str,
+    sentence: &'static str,
+    find: fn(&str) -> Option<usize>,
+}
+
+/// The rules, in the order they are tried: the first that matches is the refusal.
+const RULES: &[Rule] = &[
+    Rule {
+        name: "control",
+        sentence: "memory refused this entry: it contains a control character",
+        find: |text| first_char(text, control),
+    },
+    Rule {
+        name: "invisible",
+        sentence: "memory refused this entry: it contains an invisible or direction-changing \
+                   character",
+        find: |text| first_char(text, invisible),
+    },
+    Rule {
+        name: "private-use",
+        sentence: "memory refused this entry: it contains a private-use character",
+        find: |text| first_char(text, private_use),
+    },
+    Rule {
+        name: "noncharacter",
+        sentence: "memory refused this entry: it contains a Unicode noncharacter",
+        find: |text| first_char(text, noncharacter),
+    },
+    Rule {
+        name: "secret-github",
+        sentence: "memory refused this entry: it contains what looks like a GitHub token",
+        find: |text| find_secret(text, GITHUB),
+    },
+    Rule {
+        name: "secret-gitlab",
+        sentence: "memory refused this entry: it contains what looks like a GitLab token",
+        find: |text| find_secret(text, GITLAB),
+    },
+    Rule {
+        name: "secret-sk",
+        sentence: "memory refused this entry: it contains what looks like an API key",
+        find: |text| find_secret(text, SK),
+    },
+    Rule {
+        name: "secret-slack",
+        sentence: "memory refused this entry: it contains what looks like a Slack token",
+        find: |text| find_secret(text, SLACK),
+    },
+    Rule {
+        name: "secret-aws",
+        sentence: "memory refused this entry: it contains what looks like an AWS access key",
+        find: |text| find_secret(text, AWS),
+    },
+    Rule {
+        name: "secret-private-key",
+        sentence: "memory refused this entry: it contains a private key",
+        find: private_key,
+    },
+    Rule {
+        name: "tag",
+        sentence: "memory refused this entry: it contains chat-template markup",
+        find: listed_tag,
+    },
+];
+
+/// The first rule `text` breaks, in the order of `RULES`, or `None`.
+pub fn refuse(text: &str) -> Option<Refusal> {
+    RULES.iter().find_map(|rule| {
+        (rule.find)(text).map(|at| Refusal {
+            rule: rule.name,
+            sentence: rule.sentence,
+            at,
+        })
+    })
+}
+
+// ---- Characters ----
+
+/// Unicode 16's `Default_Ignorable_Code_Point` (DerivedCoreProperties.txt), inclusive ranges.
+const DEFAULT_IGNORABLE: &[(u32, u32)] = &[
+    (0x00AD, 0x00AD),
+    (0x034F, 0x034F),
+    (0x061C, 0x061C),
+    (0x115F, 0x1160),
+    (0x17B4, 0x17B5),
+    (0x180B, 0x180F),
+    (0x200B, 0x200F),
+    (0x202A, 0x202E),
+    (0x2060, 0x206F),
+    (0x3164, 0x3164),
+    (0xFE00, 0xFE0F),
+    (0xFEFF, 0xFEFF),
+    (0xFFA0, 0xFFA0),
+    (0xFFF0, 0xFFF8),
+    (0x1BCA0, 0x1BCA3),
+    (0x1D173, 0x1D17A),
+    (0xE0000, 0xE0FFF),
+];
+
+/// Invisible without being default-ignorable: the Braille blank, and the line and paragraph
+/// separators.
+const ALSO_INVISIBLE: &[u32] = &[0x2800, 0x2028, 0x2029];
+
+/// The private-use areas, inclusive: the one in the BMP, and planes 15 and 16 but for the last two
+/// code points of each, which are noncharacters.
+const PRIVATE_USE: &[(u32, u32)] = &[(0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD)];
+
+/// The byte offset of the first character in `text` for which `rule` is true.
+fn first_char(text: &str, rule: fn(char) -> bool) -> Option<usize> {
+    // `text.char_indices()` gives each character with its byte offset. `find` the first pair
+    // whose character `rule` accepts, and `map` it to its offset. A byte offset, not a count of
+    // characters: a test puts a two-byte character first.
+    text.char_indices()
+        .find(|&(_, c)| rule(c))
+        .map(|(at, _)| at)
+}
+
+/// A control character other than line feed and tab.
+fn control(c: char) -> bool {
+    // `c.is_control()`, and `c` is neither '\n' nor '\t'.
+    c.is_control() && c != '\n' && c != '\t'
+}
+
+/// A default-ignorable character, or one of `ALSO_INVISIBLE`.
+fn invisible(c: char) -> bool {
+    let code = u32::from(c);
+    ALSO_INVISIBLE.contains(&code)
+        || DEFAULT_IGNORABLE
+            .iter()
+            .any(|&(low, high)| (low..=high).contains(&code))
+}
+
+/// A character in one of the `PRIVATE_USE` ranges.
+fn private_use(c: char) -> bool {
+    // As `invisible` does for `DEFAULT_IGNORABLE`: `u32::from(c)` is inside one of the inclusive
+    // `(low, high)` ranges of `PRIVATE_USE`.
+    let code = u32::from(c);
+    PRIVATE_USE
+        .iter()
+        .any(|&(low, high)| (low..=high).contains(&code))
+}
+
+/// U+FDD0 to U+FDEF, and the last two code points of every plane (U+FFFE, U+FFFF, U+1FFFE, …).
+fn noncharacter(c: char) -> bool {
+    // With `code = u32::from(c)`: `(0xFDD0..=0xFDEF).contains(&code)`, or `code & 0xFFFE ==
+    // 0xFFFE`, which is true for exactly the last two code points of each of the 17 planes.
+    let code = u32::from(c);
+    (0xFDD0..=0xFDEF).contains(&code) || code & 0xFFFE == 0xFFFE
+}
+
+// ---- Secrets ----
+
+/// A secret known by its prefix: the prefix at the start of a word, then a run of key characters.
+struct Secret {
+    prefixes: &'static [&'static str],
+    /// Which characters the run is made of. The run ends at the first other character.
+    key: fn(char) -> bool,
+    /// The run's shortest and longest length, in characters, inclusive.
+    min: usize,
+    max: usize,
+    /// The run must also hold an upper-case letter and a digit: for a prefix that ordinary
+    /// kebab-case or snake_case names can start with.
+    mixed: bool,
+}
+
+// Sources, read 2026-09-26. GitHub: "Behind GitHub's new authentication token formats" (2021:
+// the five prefixes, 30 base62 characters and a 6-character checksum) and "About authentication to
+// GitHub" (`github_pat_` for fine-grained tokens, no length given); the changelog of 2026-04-24
+// (installation tokens become `ghs_`, the app id, `_` and a JWT, about 520 characters).
+const GITHUB: &[Secret] = &[
+    Secret {
+        prefixes: &["ghp_", "gho_", "ghu_", "ghs_", "ghr_"],
+        key: alphanumeric,
+        min: 36,
+        max: usize::MAX,
+        mixed: false,
+    },
+    // Its length and alphabet are not documented: 40 and `mixed` keep snake_case names out.
+    Secret {
+        prefixes: &["github_pat_"],
+        key: word,
+        min: 40,
+        max: usize::MAX,
+        mixed: true,
+    },
+    // The stateless installation token: the app id, `_`, and a JWT, which has dots.
+    Secret {
+        prefixes: &["ghs_"],
+        key: dotted,
+        min: 40,
+        max: usize::MAX,
+        mixed: true,
+    },
+];
+
+// GitLab's token overview lists the prefix `glpat-`, with no length or alphabet: 20 is the spec's
+// floor, and dots are allowed because newer tokens are said to hold them (not verified).
+const GITLAB: &[Secret] = &[Secret {
+    prefixes: &["glpat-"],
+    key: dotted,
+    min: 20,
+    max: usize::MAX,
+    mixed: false,
+}];
+
+// `sk-` keys (OpenAI's `sk-proj-`, Anthropic's `sk-ant-`; formats not verified): the spec's 20
+// characters, and `mixed`, which a random key of that length almost always is.
+const SK: &[Secret] = &[Secret {
+    prefixes: &["sk-"],
+    key: dashed,
+    min: 20,
+    max: usize::MAX,
+    mixed: true,
+}];
+
+// Slack's token types page documents `xoxb-` and `xoxp-` and a 32-character secret at the end
+// (6 or 10 in tokens before 2016); `xoxa-`, `xoxr-` and `xoxs-` are older kinds, from the spec.
+const SLACK: &[Secret] = &[Secret {
+    prefixes: &["xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"],
+    key: slack,
+    min: 20,
+    max: usize::MAX,
+    mixed: false,
+}];
+
+// IAM's "Unique identifiers" lists `AKIA` (access key) and `ASIA` (temporary access key); the IAM
+// API gives an access key id only as 16 to 128 word characters. AWS's own example key id has 16
+// after the prefix, as the spec says; a longer run is not taken for one.
+const AWS: &[Secret] = &[Secret {
+    prefixes: &["AKIA", "ASIA"],
+    key: upper_or_digit,
+    min: 16,
+    max: 16,
+    mixed: false,
+}];
+
+fn alphanumeric(c: char) -> bool {
+    c.is_ascii_alphanumeric()
+}
+
+fn word(c: char) -> bool {
+    c.is_ascii_alphanumeric() || c == '_'
+}
+
+fn dashed(c: char) -> bool {
+    c.is_ascii_alphanumeric() || c == '_' || c == '-'
+}
+
+fn dotted(c: char) -> bool {
+    c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.'
+}
+
+fn slack(c: char) -> bool {
+    c.is_ascii_alphanumeric() || c == '-'
+}
+
+fn upper_or_digit(c: char) -> bool {
+    c.is_ascii_uppercase() || c.is_ascii_digit()
+}
+
+/// The byte offset of the first place in `text` where one of `secrets` starts, or `None`.
+fn find_secret(text: &str, secrets: &[Secret]) -> Option<usize> {
+    let mut first: Option<usize> = None;
+    for secret in secrets {
+        for prefix in secret.prefixes {
+            for (at, _) in text.match_indices(prefix) {
+                if let Some((before, from)) = text.split_at_checked(at)
+                    && let Some(after) = from.strip_prefix(prefix)
+                    && starts_word(before)
+                    && key_run(after, secret)
+                {
+                    first = Some(first.map_or(at, |seen| seen.min(at)));
+                    break;
+                }
+            }
+        }
+    }
+    first
+}
+
+/// Whether a prefix right after `before` starts a word: `before` is empty, or its last character
+/// is not an ASCII letter or digit, `_` or `-`.
+fn starts_word(before: &str) -> bool {
+    // Take `before.chars().next_back()`, its last character. `None` (an empty `before`) is a
+    // start. Otherwise it is a start unless that character `is_ascii_alphanumeric()` or is '_' or
+    // '-': `!before.chars().next_back().is_some_and(|c| …)`.
+    !before
+        .chars()
+        .next_back()
+        .is_some_and(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
+}
+
+/// Whether `after` starts with a run of `secret.key` characters whose length is in
+/// `secret.min..=secret.max`, and, if `secret.mixed`, which holds an ASCII upper-case letter and
+/// an ASCII digit.
+fn key_run(after: &str, secret: &Secret) -> bool {
+    // Walk `after.chars().take_while(|&c| (secret.key)(c))`, counting the characters and noting
+    // whether any `is_ascii_uppercase()` and any `is_ascii_digit()`. True when the count is in
+    // `(secret.min..=secret.max)` (`contains(&count)`) and, if `secret.mixed`, both were seen.
+    // What follows the run does not matter.
+    let mut count = 0usize;
+    let mut has_upper = false;
+    let mut has_digit = false;
+    for c in after.chars().take_while(|&c| (secret.key)(c)) {
+        count += 1;
+        has_upper |= c.is_ascii_uppercase();
+        has_digit |= c.is_ascii_digit();
+    }
+    (secret.min..=secret.max).contains(&count) && (!secret.mixed || (has_upper && has_digit))
+}
+
+/// How a private-key block's first line ends: RFC 7468's labels and OpenSSH's (`… PRIVATE KEY`),
+/// and OpenPGP's armor header (`PGP PRIVATE KEY BLOCK`, RFC 9580 section 6.2.1).
+const PRIVATE_KEY_ENDS: &[&str] = &["PRIVATE KEY-----", "PRIVATE KEY BLOCK-----"];
+
+/// The byte offset of the first line whose text, white space aside, starts `-----BEGIN` and
+/// ends with one of `PRIVATE_KEY_ENDS`.
+fn private_key(text: &str) -> Option<usize> {
+    // For each `(at, _)` in `text.match_indices("-----BEGIN")`: `text.split_at_checked(at)?`
+    // gives `(before, from)`; the line up to `at` is `before.rsplit('\n').next()`, and the line
+    // from `at` is `from.split('\n').next()`, each `.unwrap_or_default()`. Return `Some(at)` for
+    // the first `at` where the first line `trim()`s to empty and the second, `trim_end()`ed, ends
+    // with one of `PRIVATE_KEY_ENDS` (`.iter().any(|end| line.ends_with(end))`). Else `None`.
+    for (at, _) in text.match_indices("-----BEGIN") {
+        let (before, from) = text.split_at_checked(at).unwrap_or_default();
+        let first_line = before.rsplit('\n').next().unwrap_or_default();
+        let second_line = from.split('\n').next().unwrap_or_default();
+        if first_line.trim().is_empty()
+            && PRIVATE_KEY_ENDS
+                .iter()
+                .any(|end| second_line.trim_end().ends_with(end))
+        {
+            return Some(at);
+        }
+    }
+    None
+}
+
+/// The byte offset of the first listed tag, matched exactly as `neutralise` matches.
+fn listed_tag(text: &str) -> Option<usize> {
+    // `next_tag(text)` (in `tags.rs`) gives the text before the first listed tag, the tag, and
+    // the text after it: map that to the length of the text before.
+    next_tag(text).map(|(before, _, _)| before.len())
+}

crates/bxcore/src/sanitize/tags.rs

@@ -104,7 +104,7 @@ pub fn neutralise(text: &str) -> Cow<'_, str> {
 }
 
 /// The first listed tag in `text`: the text before it, the tag, and the text after it.
-fn next_tag(text: &str) -> Option<(&str, &'static Tag, &str)> {
+pub(super) fn next_tag(text: &str) -> Option<(&str, &'static Tag, &str)> {
     // For each `(at, _)` of `text.match_indices('<')`: `let (before, from) =
     // text.split_at_checked(at)?;`, then `TAGS.iter().find(|tag| from.starts_with(tag.text))`.
     // If it finds one: `let after = from.get(tag.text.len()..)?;` and return