p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
2026-09-22 21:54:01 -07:00
parent 6cb206fb27
commit 9444a35322
311 changed files with 238557 additions and 92 deletions
@@ -0,0 +1,100 @@
+/*!include:re2c "base.re" */
+
+// Original pattern: (?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story
+func IsPositiveClass(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for { /*!use:re2c:base_template
+		re2c:case-insensitive = 1;
+
+		positive = article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story;
+
+		{positive} { return true }
+		*          { continue }
+		$          { return false }
+		*/
+	}
+}
+
+// For IsNegativeClass, its original pattern is like this:
+// (?i)-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget
+//
+// Unfortunately, re2go doesn't handle anchor like ^ and $ internally, so for convenience
+// I'll split that pattern into two:
+// - `^hid$| hid$| hid |^hid `
+// - `-ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget`
+func IsNegativeClass(input string) bool {
+	return isNegativeClass1(input) || isNegativeClass2(input)
+}
+
+// This one handle: `^hid$| hid$| hid |^hid `
+func isNegativeClass1(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	// Variable for capturing parentheses (twice the number of groups).
+	/*!maxnmatch:re2c*/
+	yypmatch := make([]int, YYMAXNMATCH*2)
+	var yynmatch int
+	_ = yynmatch
+
+	// Autogenerated tag variables used by the lexer to track tag values.
+	/*!stags:re2c format = 'var @@ int; _ = @@\n'; */
+
+	for { /*!use:re2c:base_template
+		re2c:posix-captures   = 1;
+		re2c:case-insensitive = 1;
+
+		(.)?hid(.)? {
+			// Extract submatch first
+			var sub1, sub2 string
+			start, end := yypmatch[0], yypmatch[1]
+
+			if yypmatch[2] != -1 {
+				sub1 = input[yypmatch[2]:yypmatch[3]]
+			}
+
+			if yypmatch[4] != -1 {
+				sub2 = input[yypmatch[4]:yypmatch[5]]
+			}
+
+			case1 := start == 0 && end == limit && sub1 == "" && sub2 == "" // '^hid$'
+			case2 := end == limit && sub1 == " " && sub2 == "" // ' hid$'
+			case3 := start == 0 && sub1 == "" && sub2 == " " // '^hid '
+			case4 := sub1 == " " && sub2 == " " // ' hid '
+
+			if case1 || case2 || case3 || case4 {
+				return true
+			}
+			continue
+		}
+
+		* { continue }
+		$ { return false }
+		*/
+	}
+}
+
+// This one handle: `-ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget`
+func isNegativeClass2(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for { /*!use:re2c:base_template
+		re2c:case-insensitive = 1;
+
+		negative = -ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget;
+
+		{negative} { return true }
+		*          { continue }
+		$          { return false }
+		*/
+	}
+}