p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
2026-09-22 21:54:01 -07:00
parent 6cb206fb27
commit 9444a35322
311 changed files with 238557 additions and 92 deletions
@@ -0,0 +1,61 @@
+package textutils
+
+import "strings"
+
+func CalculateCodeFenceOccurrences(fenceChar rune, content string) int {
+	var occurrences []int
+
+	var charsTogether int
+	for _, char := range content {
+		// We encountered a fence character, now count how many
+		// are directly afterwards
+		if char == fenceChar {
+			charsTogether++
+		} else if charsTogether != 0 {
+			occurrences = append(occurrences, charsTogether)
+			charsTogether = 0
+		}
+	}
+
+	// If the last element in the content was a fenceChar
+	if charsTogether != 0 {
+		occurrences = append(occurrences, charsTogether)
+	}
+
+	return findMax(occurrences)
+}
+
+// CalculateCodeFence can be passed the content of a code block and it returns
+// how many fence characters (` or ~) should be used.
+//
+// This is useful if the html content includes the same fence characters
+// for example ```
+// -> https://stackoverflow.com/a/49268657
+func CalculateCodeFence(fenceChar rune, content string) string {
+	repeat := CalculateCodeFenceOccurrences(fenceChar, content)
+
+	// The outer fence block always has to have
+	// at least one character more than any content inside
+	repeat++
+
+	// You have to have at least three fence characters
+	// to be recognized as a code block
+	if repeat < 3 {
+		repeat = 3
+	}
+
+	return strings.Repeat(string(fenceChar), repeat)
+}
+
+func findMax(a []int) (max int) {
+	for i, value := range a {
+		if i == 0 {
+			max = a[i]
+		}
+
+		if value > max {
+			max = value
+		}
+	}
+	return max
+}
@@ -0,0 +1,31 @@
+package textutils
+
+import (
+	"bytes"
+)
+
+func CollapseInlineCodeContent(content []byte) []byte {
+	// TODO: what about other characters like the reset char? Maybe unicode.IsSpace?
+	content = bytes.ReplaceAll(content, []byte{'\n'}, []byte{' '})
+	content = bytes.ReplaceAll(content, []byte{'\t'}, []byte{' '})
+
+	content = bytes.TrimSpace(content)
+
+	newChars := make([]byte, 0, len(content))
+
+	var count int
+	for _, char := range content {
+		if char == ' ' {
+			count++
+		} else {
+			count = 0
+		}
+
+		if count > 1 {
+			continue
+		}
+		newChars = append(newChars, char)
+	}
+
+	return newChars
+}
@@ -0,0 +1,139 @@
+package textutils
+
+import (
+	"bytes"
+	"unicode/utf8"
+)
+
+func TrimUnnecessaryHardLineBreaks(content []byte) []byte {
+	content = bytes.ReplaceAll(content, []byte("  \n\n"), []byte("\n\n"))
+	content = bytes.ReplaceAll(content, []byte("  \n  \n"), []byte("\n\n"))
+	content = bytes.ReplaceAll(content, []byte("  \n \n"), []byte("\n\n"))
+	// out = bytes.ReplaceAll(out, []byte("\n  \n"), []byte("\n\n"))
+
+	return content
+}
+
+func TrimConsecutiveNewlines(input []byte) []byte {
+	var result []byte
+	newlineCount := 0
+	spaceBuffer := []byte{}
+
+	for i := 0; i < len(input); {
+		r, size := utf8.DecodeRune(input[i:])
+
+		if r == '\n' {
+			newlineCount++
+			if newlineCount <= 2 {
+				// Preserve up to 2 newlines, including preceding spaces
+				result = append(result, spaceBuffer...)
+				result = append(result, '\n')
+				spaceBuffer = spaceBuffer[:0] // Clear space buffer
+			} else {
+				// Skip additional newlines
+				spaceBuffer = spaceBuffer[:0] // Clear space buffer
+			}
+		} else if r == ' ' {
+			// Collect spaces into the space buffer
+			spaceBuffer = append(spaceBuffer, input[i:i+size]...)
+		} else {
+			// Reset newline count and append non-newline characters
+			newlineCount = 0
+			result = append(result, spaceBuffer...)
+			result = append(result, input[i:i+size]...)
+			spaceBuffer = spaceBuffer[:0] // Clear space buffer
+		}
+
+		i += size
+	}
+
+	// Append any trailing spaces
+	result = append(result, spaceBuffer...)
+
+	return result
+}
+
+/*
+func TrimConsecutiveNewlines(source []byte) []byte {
+	// Some performance optimizations:
+	// - If no replacement was done, we return the original slice and don't allocate.
+	// - We batch appends
+
+	var ret []byte
+
+	startNormal := 0
+	startMatch := -1
+
+	count := 0
+	// for i, b := range source {
+	for i := 0; i < len(source); i++ {
+		r, size := utf8.DecodeRune(source[i:])
+		_ = size
+
+		isNewline := r == '\n' // || r == marker.MarkerLineBreak
+		if isNewline {
+			count += 1
+		}
+
+		if startMatch == -1 && isNewline {
+			// Start of newlines
+			startMatch = i
+			i = i + size - 1
+			continue
+		} else if startMatch != -1 && isNewline {
+			// Middle of newlines
+			i = i + size - 1
+			continue
+		} else if startMatch != -1 {
+			// Character after the last newline character
+
+			if count > 2 {
+				if ret == nil {
+					ret = make([]byte, 0, len(source))
+				}
+
+				ret = append(ret, source[startNormal:startMatch]...)
+				ret = append(ret, '\n', '\n')
+				startNormal = i
+			}
+
+			startMatch = -1
+			count = 0
+		}
+	}
+
+	getStartEnd := func() (int, int, bool, bool) {
+		if startMatch == -1 && startNormal == 0 {
+			// a) no changes need to be done
+			return -1, -1, false, false
+		}
+
+		if count <= 2 {
+			// b) Only the normal characters still need to be added
+			return startNormal, len(source), true, false
+		}
+
+		// c) The match still needs to be replaced (and possible the previous normal characters be added)
+		return startNormal, startMatch, true, true
+	}
+
+	start, end, isKeepNeeded, isReplaceNeeded := getStartEnd()
+	if isKeepNeeded {
+		if ret == nil {
+			ret = make([]byte, 0, len(source))
+		}
+
+		ret = append(ret, source[start:end]...)
+		if isReplaceNeeded {
+			ret = append(ret, '\n', '\n')
+		}
+	}
+
+	if ret == nil {
+		// Hurray, we did not do any allocations with make()
+		// and instead just return the original slice.
+		return source
+	}
+	return ret
+}
+*/
@@ -0,0 +1,37 @@
+package textutils
+
+import (
+	"bytes"
+)
+
+// DelimiterForEveryLine puts the delimiter not just at the start and end of the string
+// but if the text is divided on multiple lines, puts the delimiters on every line with content.
+//
+// Otherwise the bold/italic delimiters won't be recognized if it contains new line characters.
+func DelimiterForEveryLine(text []byte, delimiter []byte) []byte {
+	var buf bytes.Buffer
+
+	lines := bytes.Split(text, []byte("\n"))
+	for i, line := range lines {
+		leftExtra, trimmed, rightExtra := SurroundingSpaces(line)
+
+		if trimmed == nil {
+			// For empty lines, we don't need a delimiter
+			buf.Write(leftExtra)
+			buf.Write(rightExtra)
+		} else {
+			buf.Write(leftExtra)
+			buf.Write(delimiter)
+			buf.Write(trimmed)
+			buf.Write(delimiter)
+			buf.Write(rightExtra)
+		}
+
+		// To join the lines again, add a newlines character
+		if i < len(lines)-1 {
+			buf.WriteRune('\n')
+		}
+	}
+
+	return buf.Bytes()
+}
@@ -0,0 +1,56 @@
+package textutils
+
+import (
+	"bytes"
+	"unicode"
+)
+
+var (
+	doubleSpace = []byte{' ', ' '}
+
+	newlineBreak              = []byte{'\n'}
+	hardLineBreak             = []byte{' ', ' ', '\n'}
+	escapedNoContentLineBreak = []byte{'\\', '\n'}
+)
+
+// EscapeMultiLine deals with multiline content inside a link or a heading.
+func EscapeMultiLine(content []byte) []byte {
+	parts := bytes.Split(content, newlineBreak)
+	if len(parts) == 1 {
+		return content
+	}
+
+	output := make([]byte, 0, len(content))
+	for i := range parts {
+		trimmedLeft := bytes.TrimLeftFunc(parts[i], unicode.IsSpace)
+
+		if len(trimmedLeft) == 0 {
+			// A blank line would interrupt the link.
+			// So we need to escape the line
+			output = append(output, escapedNoContentLineBreak...)
+			continue
+		}
+
+		isLast := i == len(parts)-1
+		if isLast {
+			// For the last line we don't need to add any "\n" anymore
+			output = append(output, trimmedLeft...)
+			continue
+		}
+
+		// Now decide what ending we want:
+		if bytes.HasSuffix(trimmedLeft, doubleSpace) {
+			// We already have "  " so adding a "\n" is enough
+			output = append(output, trimmedLeft...)
+			output = append(output, newlineBreak...)
+			continue
+		} else {
+			// We *prefer* having a hard-line-break "  \n"
+			output = append(output, trimmedLeft...)
+			output = append(output, hardLineBreak...)
+			continue
+		}
+	}
+
+	return output
+}
@@ -0,0 +1,16 @@
+package textutils
+
+func PrefixLines(source []byte, repl []byte) []byte {
+	newSlice := make([]byte, 0, len(source))
+
+	newSlice = append(newSlice, repl...)
+	for _, b := range source {
+		newSlice = append(newSlice, b)
+		if b == '\n' {
+			newSlice = append(newSlice, repl...)
+		}
+
+	}
+
+	return newSlice
+}
@@ -0,0 +1,39 @@
+package textutils
+
+import "bytes"
+
+const (
+	DOUBLE_QUOTE = '"'
+	SINGLE_QUOTE = '\''
+)
+
+func SurroundBy(content []byte, chars []byte) []byte {
+	content = append(chars, content...)
+	content = append(content, chars...)
+	return content
+}
+func SurroundByQuotes(content []byte) []byte {
+	if len(content) == 0 {
+		return nil
+	}
+
+	containsDoubleQuote := bytes.ContainsRune(content, DOUBLE_QUOTE)
+	containsSingleQuote := bytes.ContainsRune(content, SINGLE_QUOTE)
+
+	if containsDoubleQuote && containsSingleQuote {
+		// Escape all quotes
+		content = bytes.ReplaceAll(content, []byte(`"`), []byte(`\"`))
+
+		// Surround the content by double quotes
+		return SurroundBy(content, []byte(`"`))
+	}
+	if containsDoubleQuote {
+		// Since it contains double quotes (but no single quotes)
+		// we can surround it by single quotes
+		return SurroundBy(content, []byte(`'`))
+	}
+
+	// It may contain single quotes, but definitely no double quotes,
+	// so we can safely surround it by double quotes.
+	return SurroundBy(content, []byte(`"`))
+}
@@ -0,0 +1,58 @@
+/*
+
+The logic to handle whitespace around delimiters was initially developed
+in the fork from "anyproto" by Roman Khafizianov and Mikhail.
+
+The changes were then merged upstream by Johannes Kaufmann.
+
+https://github.com/anyproto/html-to-markdown
+https://github.com/JohannesKaufmann/html-to-markdown
+
+-----------
+
+MIT License
+
+Copyright (c) 2018 Johannes Kaufmann
+Copyright (c) 2020 Roman Khafizianov
+Copyright (c) 2023 Mikhail
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+package textutils
+
+import (
+	"bytes"
+	"unicode"
+)
+
+func SurroundingSpaces(content []byte) ([]byte, []byte, []byte) {
+	rightTrimmed := bytes.TrimRightFunc(content, func(r rune) bool {
+		return unicode.IsSpace(r)
+	})
+	rightExtra := content[len(rightTrimmed):]
+
+	trimmed := bytes.TrimLeftFunc(rightTrimmed, func(r rune) bool {
+		return unicode.IsSpace(r)
+	})
+	leftExtra := content[0 : len(rightTrimmed)-len(trimmed)]
+
+	return leftExtra, trimmed, rightExtra
+}