//! The rules of M5b spec, section 6.1, at their edges: which characters each character rule //! covers, each secret's shortest length and word start, the order the rules are tried in, their //! sentences and offsets. Tags are built from their names (`support/tags.rs`), and every fake //! secret from pieces (`support/keys.rs`). No assertion prints a text. Do not edit. #[path = "support/keys.rs"] mod keys; #[path = "support/tags.rs"] mod tags; use bxcore::sanitize::refuse; use keys::{caps, lower, mixed, pem, rule}; use tags::{bar, close, measured}; fn around(code: u32) -> String { format!("before{}after", char::from_u32(code).unwrap()) } #[test] fn control_characters_are_refused_but_line_feed_and_tab_are_not() { for code in [ 0x00, 0x07, 0x08, 0x0B, 0x0C, 0x0D, 0x1B, 0x7F, 0x80, 0x85, 0x9F, ] { assert_eq!(rule(&around(code)), Some("control"), "{code:#x}"); } assert_eq!(rule("one\ntwo\tthree\n"), None); } #[test] fn invisible_and_direction_changing_characters_are_refused() { for code in [ 0x00AD, 0x034F, 0x061C, 0x115F, 0x1160, 0x17B4, 0x180E, 0x200B, 0x200D, 0x200E, 0x202A, 0x202E, 0x2060, 0x2066, 0x2069, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, 0x1D173, 0x1D17A, 0xE0000, 0xE0001, 0xE0041, 0xE007F, 0xE0100, 0xE01EF, 0xE0FFF, 0x2800, 0x2028, 0x2029, ] { assert_eq!(rule(&around(code)), Some("invisible"), "{code:#x}"); } for code in [ 0x0020, 0x00A0, 0x2010, 0x2070, 0x3165, 0xFE10, 0xFFF9, 0xE1000, ] { assert_eq!(rule(&around(code)), None, "{code:#x}"); } } #[test] fn private_use_characters_are_refused() { for code in [ 0xE000, 0xE001, 0xF8FF, 0xF0000, 0xF8000, 0xFFFFD, 0x100000, 0x108000, 0x10FFFD, ] { assert_eq!(rule(&around(code)), Some("private-use"), "{code:#x}"); } for code in [0xD7FF, 0xF900, 0xEFFFD] { assert_eq!(rule(&around(code)), None, "{code:#x}"); } } #[test] fn noncharacters_are_refused_in_every_plane() { for code in 0xFDD0..=0xFDEF { assert_eq!(rule(&around(code)), Some("noncharacter"), "{code:#x}"); } for plane in 0..=16u32 { for last in [0xFFFE, 0xFFFF] { let code = plane * 0x10000 + last; assert_eq!(rule(&around(code)), Some("noncharacter"), "{code:#x}"); } } for code in [0xFDCF, 0xFDF0, 0xFFFD, 0x1FFFD, 0x2FFFD] { assert_eq!(rule(&around(code)), None, "{code:#x}"); } } #[test] fn every_listed_tag_is_refused() { for (tag, id) in measured() { assert_eq!(rule(&format!("a{tag}b")), Some("tag"), "{id}"); assert_eq!(rule(&tag), Some("tag"), "{id}"); } } /// Review focus 1, for memory: a tag is matched exactly as `neutralise` matches it. The shapes that /// are not the tag's token pass, except the one split by an invisible character, which is refused /// for the character. #[test] fn tag_shapes_are_decided() { for (tag, id) in measured() { assert_eq!( rule(&format!("x{tag}y")), Some("tag"), "{id}: inside a word" ); assert_eq!(rule(&tag.to_uppercase()), None, "{id}: upper case"); let split = [&tag[..2], "\u{200B}", &tag[2..]].concat(); assert_eq!( rule(&split), Some("invisible"), "{id}: split by a zero-width space" ); let spaced = [&tag[..2], " ", &tag[2..]].concat(); assert_eq!(rule(&spaced), None, "{id}: split by a space"); let wide = tag.replace('<', "\u{FF1C}").replace('>', "\u{FF1E}"); assert_eq!(rule(&wide), None, "{id}: full-width brackets"); let escaped = tag.replace('<', "<").replace('>', ">"); assert_eq!(rule(&escaped), None, "{id}: neutralised"); } } /// M5b spec, section 14.1 (accepted): the variation selectors and the zero-width joiner stay /// refused, so emoji built with them are refused. #[test] fn emoji_with_a_variation_selector_or_a_joiner_are_refused() { for (name, text) in [ ("a red heart", "\u{2764}\u{FE0F}"), ("a keycap", "1\u{FE0F}\u{20E3}"), ("a technologist", "\u{1F468}\u{200D}\u{1F4BB}"), ] { assert_eq!(rule(text), Some("invisible"), "{name}"); } } /// Each secret at its shortest length is refused, and one character shorter passes. An AWS key id /// is exactly 16 characters after its prefix. #[test] fn secrets_are_refused_from_their_minimum_length() { let github_jwt = |n: usize| format!("ghs_{}.{}", mixed(20), mixed(n - 21)); let cases: Vec<(&str, String, String, &str)> = vec![ ( "ghp_", format!("ghp_{}", mixed(36)), format!("ghp_{}", mixed(35)), "secret-github", ), ( "gho_", format!("gho_{}", mixed(36)), format!("gho_{}", mixed(35)), "secret-github", ), ( "ghu_", format!("ghu_{}", mixed(36)), format!("ghu_{}", mixed(35)), "secret-github", ), ( "ghs_", format!("ghs_{}", mixed(36)), format!("ghs_{}", mixed(35)), "secret-github", ), ( "ghr_", format!("ghr_{}", mixed(36)), format!("ghr_{}", mixed(35)), "secret-github", ), ( "github_pat_", format!("github_pat_{}", mixed(40)), format!("github_pat_{}", mixed(39)), "secret-github", ), ( "ghs_ with a dot", github_jwt(40), github_jwt(39), "secret-github", ), ( "glpat-", format!("glpat-{}", lower(20)), format!("glpat-{}", lower(19)), "secret-gitlab", ), ( "sk-", format!("sk-{}", mixed(20)), format!("sk-{}", mixed(19)), "secret-sk", ), ( "xoxb-", format!("xoxb-{}", lower(20)), format!("xoxb-{}", lower(19)), "secret-slack", ), ( "AKIA", format!("AKIA{}", caps(16)), format!("AKIA{}", caps(15)), "secret-aws", ), ]; for (name, long_enough, too_short, want) in cases { assert_eq!(rule(&long_enough), Some(want), "{name}: at the minimum"); assert_eq!( rule(&format!("{long_enough} end")), Some(want), "{name}: before a space" ); assert_eq!(rule(&too_short), None, "{name}: one short"); } assert_eq!(rule(&format!("AKIA{}", caps(17))), None, "AKIA: one long"); assert_eq!( rule(&format!("ASIA{} end", caps(17))), None, "ASIA: one long" ); let camel = format!("AKIA{}", mixed(16)); assert_eq!(rule(&camel), None, "AKIA: lower case is not a key id"); } /// `sk-`, `github_pat_` and the dotted `ghs_` form are also ordinary name prefixes, so their run /// must hold an upper-case letter and a digit. #[test] fn generic_prefixes_need_an_upper_case_letter_and_a_digit() { for prefix in ["sk-", "github_pat_", "ghs_x."] { let lower_only = format!("{prefix}{}", lower(60)); let no_digit = format!("{prefix}AbCdEfGhIj{}", lower(50)); let no_upper = format!("{prefix}ab1cd2ef3g{}", lower(50)); let both = format!("{prefix}Ab1{}", lower(57)); assert_eq!(rule(&lower_only), None, "{prefix}: lower case only"); assert_eq!(rule(&no_digit), None, "{prefix}: no digit"); assert_eq!(rule(&no_upper), None, "{prefix}: no upper case"); assert!(rule(&both).is_some(), "{prefix}: both"); } // The others need neither: a GitLab or Slack token can be lower case and digits only. assert_eq!(rule(&format!("glpat-{}", lower(24))), Some("secret-gitlab")); assert_eq!(rule(&format!("xoxp-{}", lower(24))), Some("secret-slack")); assert_eq!(rule(&format!("ghp_{}", lower(36))), Some("secret-github")); } #[test] fn a_secret_prefix_must_start_a_word() { let keys = [ (format!("ghp_{}", mixed(36)), "secret-github"), (format!("github_pat_{}", mixed(82)), "secret-github"), (format!("glpat-{}", mixed(20)), "secret-gitlab"), (format!("sk-{}", mixed(48)), "secret-sk"), (format!("xoxb-{}", mixed(40)), "secret-slack"), (format!("AKIA{}", caps(16)), "secret-aws"), ]; for (key, want) in &keys { for before in ["", " ", "=", "\"", "'", "/", ":", "(", "\n", "`", "é"] { assert_eq!( rule(&format!("{before}{key}")), Some(*want), "{want} after {before:?}" ); } for before in ["a", "Z", "9", "_", "-"] { assert_eq!( rule(&format!("{before}{key}")), None, "{want} after {before:?}" ); } } } #[test] fn a_private_key_header_must_be_a_line_of_its_own() { let header = pem("BEGIN RSA PRIVATE KEY"); for (name, text) in [ ("first", header.clone()), ("after a line", format!("key:\n{header}\nMII")), ("indented", format!("key: |\n \t {header}\n")), ("with trailing spaces", format!("{header} \nMII")), ( "a PGP private key block, indented", format!(" {} \n", pem("BEGIN PGP PRIVATE KEY BLOCK")), ), ] { assert_eq!(rule(&text), Some("secret-private-key"), "{name}"); } for (name, text) in [ ("inside a line", format!("see {header} there")), ("after text on its line", format!("key:{header}")), ( "without the closing dashes", "-----BEGIN RSA PRIVATE KEY".to_string(), ), ("a public key", pem("BEGIN RSA PUBLIC KEY")), ("a PGP public key block", pem("BEGIN PGP PUBLIC KEY BLOCK")), ( "a PGP block without the closing dashes", "-----BEGIN PGP PRIVATE KEY BLOCK".to_string(), ), ("an end line", pem("END RSA PRIVATE KEY")), ] { assert_eq!(rule(&text), None, "{name}"); } } /// One text per rule, each breaking that rule alone, in the order the rules are tried. fn one_per_rule() -> Vec<(&'static str, String)> { vec![ ("control", "\u{7}".to_string()), ("invisible", "\u{200B}".to_string()), ("private-use", "\u{E000}".to_string()), ("noncharacter", "\u{FFFE}".to_string()), ("secret-github", format!("ghp_{}", mixed(36))), ("secret-gitlab", format!("glpat-{}", mixed(20))), ("secret-sk", format!("sk-{}", mixed(24))), ("secret-slack", format!("xoxb-{}", lower(24))), ("secret-aws", format!("AKIA{}", caps(16))), ("secret-private-key", pem("BEGIN RSA PRIVATE KEY")), ("tag", bar("im_end")), ] } /// Every pair of rules, with the later rule's text first: the earlier rule is named, wherever it /// is in the text. #[test] fn the_first_rule_in_order_is_named() { let rules = one_per_rule(); for (name, text) in &rules { assert_eq!(rule(text), Some(*name), "{name} alone"); } for (i, (first, first_text)) in rules.iter().enumerate() { for (later, later_text) in rules.iter().skip(i + 1) { let both = format!("{later_text}\n{first_text}"); assert_eq!(rule(&both), Some(*first), "{first} before {later}"); } } } /// The sentences are fixed, and none quotes the text. #[test] fn each_rule_has_its_sentence() { let sentences = [ "memory refused this entry: it contains a control character", "memory refused this entry: it contains an invisible or direction-changing character", "memory refused this entry: it contains a private-use character", "memory refused this entry: it contains a Unicode noncharacter", "memory refused this entry: it contains what looks like a GitHub token", "memory refused this entry: it contains what looks like a GitLab token", "memory refused this entry: it contains what looks like an API key", "memory refused this entry: it contains what looks like a Slack token", "memory refused this entry: it contains what looks like an AWS access key", "memory refused this entry: it contains a private key", "memory refused this entry: it contains chat-template markup", ]; let rules = one_per_rule(); assert_eq!(rules.len(), sentences.len()); for ((name, text), sentence) in rules.iter().zip(sentences) { let refusal = refuse(text).unwrap(); assert_eq!(refusal.rule, *name); assert_eq!(refusal.sentence, sentence, "{name}"); } } /// `at` is the byte offset where the rule matched: the character, the prefix, the header line's /// dashes, the tag's first bracket. #[test] fn at_is_where_the_rule_matched() { let cases: Vec<(&str, String, usize)> = vec![ ("a control character", "ab\u{7}".to_string(), 2), ("after a two-byte character", "é\u{200B}".to_string(), 2), ("a private-use character", "日\u{E000}".to_string(), 3), ("a prefix", format!("key: ghp_{}", mixed(36)), 5), ( "the first occurrence that is a key", format!("sk-learn sk-{}", mixed(24)), 9, ), ( "the earliest of several prefixes", format!("a ghu_{} b ghp_{}", mixed(36), mixed(36)), 2, ), ( "an indented header", format!("x\n {}", pem("BEGIN EC PRIVATE KEY")), 4, ), ("a tag", format!("abc{}", close("think")), 3), ]; for (name, text, at) in cases { assert_eq!(refuse(&text).map(|refusal| refusal.at), Some(at), "{name}"); } }