p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification
is ignored. No 502. keys add writes the secret only to --out. Release binary
has no -test-allow-prefix.

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/add_space.go

@@ -0,0 +1,66 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func getFirstChildNode(startNode *html.Node, matchFn func(n *html.Node) bool) *html.Node {
+	node := startNode.FirstChild
+	for node != nil {
+		name := dom.NodeName(node)
+		if name == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetNextNeighborNode(node)
+		} else if matchFn(node) {
+			return node
+		} else {
+			return nil
+		}
+	}
+
+	return nil
+}
+
+func getLastChildNode(startNode *html.Node, matchFn func(n *html.Node) bool) *html.Node {
+	node := startNode.LastChild
+	for node != nil {
+		name := dom.NodeName(node)
+		if name == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetPrevNeighborNode(node)
+		} else if matchFn(node) {
+			return node
+		} else {
+			return nil
+		}
+	}
+
+	return nil
+}
+func AddSpace(ctx context.Context, doc *html.Node, isOuterNode, isInnerNode func(*html.Node) bool) {
+	node := doc
+	for node != nil {
+		if isOuterNode(node) {
+			firstChild := getFirstChildNode(node, isInnerNode)
+			if firstChild != nil {
+				prev := getPrevTextNode(node)
+				if prev != nil {
+					prev.Data = prev.Data + " "
+				}
+			}
+
+			lastChild := getLastChildNode(node, isInnerNode)
+			if lastChild != nil {
+				next := getNextTextNode(node)
+				if next != nil {
+					next.Data = " " + next.Data
+				}
+			}
+		}
+
+		node = dom.GetNextNeighborElement(node)
+	}
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/adjacent.go

@@ -0,0 +1,75 @@
+package domutils
+
+import (
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func collectAdjacentNodes(node *html.Node, matchFn func(n *html.Node) bool) []*html.Node {
+	var collectedNodes []*html.Node
+
+	node = node.NextSibling
+	for node != nil {
+		name := dom.NodeName(node)
+		if name == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetNextNeighborNode(node)
+		} else if matchFn(node) {
+			collectedNodes = append(collectedNodes, node)
+			node = dom.GetNextNeighborNodeExcludingOwnChild(node)
+		} else {
+			// Return the collected nodes
+			return collectedNodes
+		}
+	}
+
+	return collectedNodes
+}
+
+func mergeChildren(destinationNode *html.Node, nodes ...*html.Node) {
+	for _, node := range nodes {
+		// We move all the children to the `destinationNode`.
+		children := dom.AllChildNodes(node)
+		for _, child := range children {
+			dom.RemoveNode(child)
+			destinationNode.AppendChild(child)
+		}
+		dom.RemoveNode(node)
+	}
+}
+
+func MergeAdjacent(doc *html.Node, matchFn func(*html.Node) bool) {
+	node := doc
+
+	for node != nil {
+		if matchFn(node) {
+			nextNodes := collectAdjacentNodes(node, matchFn)
+
+			mergeChildren(node, nextNodes...)
+		}
+
+		node = dom.GetNextNeighborElement(node)
+	}
+}
+
+// - - - - - - - - //
+
+func MergeAdjacentTextNodes(n *html.Node) {
+	if n == nil {
+		return
+	}
+
+	var prev *html.Node
+	for c := n.FirstChild; c != nil; {
+		next := c.NextSibling
+		if c.Type == html.TextNode && prev != nil && prev.Type == html.TextNode {
+			// Combine adjacent text nodes
+			prev.Data += c.Data
+			n.RemoveChild(c)
+		} else {
+			MergeAdjacentTextNodes(c)
+			prev = c
+		}
+		c = next
+	}
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/alternatives.go

@@ -0,0 +1,114 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+// TODO: make this configurable via the options???
+func getMarkdownStructure(name string) string {
+	switch name {
+	case "#document", "html", "head", "body",
+		"blockquote", "ul", "ol", "li":
+		// A container block can also contain other blocks.
+		return "container_block"
+
+	// Note: "p" would also be part of "leaf_block"
+	case "hr", "pre",
+		"h1", "h2", "h3", "h4", "h5", "h6":
+		// Leaf blocks can contain inline content
+		// but NOT other blocks.
+		return "leaf_block"
+
+	case "#text", "span", "code",
+		"b", "strong", "i", "em",
+		"a", "img", "br":
+		return "inline"
+
+	case "div", "p":
+		// Since these are just placing newlines,
+		// we dont categorize them.
+		return ""
+
+	default:
+		return ""
+	}
+}
+
+func headingAlternative(ctx context.Context, node *html.Node) {
+	node.Data = "strong"
+
+	newChild := &html.Node{
+		Type: html.ElementNode,
+		Data: "br",
+	}
+	node.Parent.InsertBefore(newChild, node.NextSibling)
+}
+func blockquoteAlternative(ctx context.Context, node *html.Node) {
+	newBefore := &html.Node{Type: html.TextNode, Data: ` "`}
+	node.Parent.InsertBefore(newBefore, node)
+
+	node.Data = "span"
+
+	newAfter := &html.Node{Type: html.TextNode, Data: `" `}
+	node.Parent.InsertBefore(newAfter, node.NextSibling)
+}
+func preAlternative(ctx context.Context, node *html.Node) {
+	node.Data = "code"
+}
+func hrAlternative(ctx context.Context, node *html.Node) {
+	dom.RemoveNode(node)
+}
+
+// TODO: make this configurable via the options?
+var alternatives = map[string]func(ctx context.Context, node *html.Node){
+	"h1":         headingAlternative,
+	"h2":         headingAlternative,
+	"h3":         headingAlternative,
+	"h4":         headingAlternative,
+	"h5":         headingAlternative,
+	"h6":         headingAlternative,
+	"blockquote": blockquoteAlternative,
+	"pre":        preAlternative,
+	"hr":         hrAlternative,
+}
+
+func LeafBlockAlternatives(ctx context.Context, doc *html.Node) {
+	var finder func(node *html.Node, isInsideLeafBlock bool, isInsideInline bool)
+	finder = func(node *html.Node, isInsideLeafBlock bool, isInsideInline bool) {
+		name := dom.NodeName(node)
+
+		structure := getMarkdownStructure(name)
+		if (structure == "container_block" || structure == "leaf_block") && (isInsideLeafBlock || isInsideInline) {
+			// A block inside an inline OR a block inside a leaf-block
+			// is not valid markdown so cannot be rendered.
+			//
+			// For example, you cannot place a blockquote inside a heading.
+			//
+			// Instead of this weird output (## Heading > My Quote)
+			// we try to find alternatives (## Heading "My Quote")
+			fn, ok := alternatives[name]
+			if ok {
+				fn(ctx, node)
+			} else {
+				node.Data = "span"
+			}
+		}
+
+		// - - - - - - - - - - - - - - - - - - - - - - //
+
+		if structure == "leaf_block" {
+			isInsideLeafBlock = true
+		}
+		if structure == "inline" {
+			isInsideInline = true
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			defer finder(child, isInsideLeafBlock, isInsideInline)
+		}
+	}
+	finder(doc, false, false)
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/domutils.go

@@ -0,0 +1,43 @@
+package domutils
+
+import (
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func getNextTextNode(startNode *html.Node) *html.Node {
+	node := dom.GetNextNeighborNodeExcludingOwnChild(startNode)
+
+	for node != nil {
+		if node.Type == html.TextNode {
+			return node
+		}
+
+		if dom.NodeName(node) == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetNextNeighborNode(node)
+			continue
+		}
+
+		return nil
+	}
+	return nil
+}
+func getPrevTextNode(startNode *html.Node) *html.Node {
+	node := dom.GetPrevNeighborNodeExcludingOwnChild(startNode)
+
+	for node != nil {
+		if node.Type == html.TextNode {
+			return node
+		}
+
+		if dom.NodeName(node) == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetPrevNeighborNode(node)
+			continue
+		}
+
+		return nil
+	}
+	return nil
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/empty_code.go

@@ -0,0 +1,42 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func hasTextChildNodes(startNode *html.Node) bool {
+	var found bool
+
+	var finder func(*html.Node)
+	finder = func(node *html.Node) {
+		if node.Type == html.TextNode && node.Data != "" {
+			found = true
+			return
+		}
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+	finder(startNode)
+
+	return found
+}
+
+func RemoveEmptyCode(ctx context.Context, doc *html.Node) {
+	node := doc
+	for node != nil {
+		if dom.NodeName(node) == "code" && !hasTextChildNodes(node) {
+			next := dom.GetNextNeighborNodeExcludingOwnChild(node)
+
+			dom.RemoveNode(node)
+
+			node = next
+			continue
+		}
+
+		node = dom.GetNextNeighborNode(node)
+	}
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/list_end_comment.go

@@ -0,0 +1,74 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+var ListEndCommentData = "THE END"
+
+func AddListEndComments(ctx context.Context, doc *html.Node) {
+	node := doc
+	for node != nil {
+		if nameIsList(node) && nextNameIsList(node) {
+			insertComment(node)
+		}
+
+		node = dom.GetNextNeighborElement(node)
+	}
+}
+
+func nameIsList(node *html.Node) bool {
+	name := dom.NodeName(node)
+	return name == "ul" || name == "ol"
+}
+
+func insertComment(listNode *html.Node) {
+	comment := &html.Node{
+		Type: html.CommentNode,
+		Data: ListEndCommentData,
+	}
+	listNode.Parent.InsertBefore(comment, listNode.NextSibling)
+}
+
+func nextNameIsList(startNode *html.Node) bool {
+	node := dom.GetNextNeighborNodeExcludingOwnChild(startNode)
+
+	for node != nil {
+		name := dom.NodeName(node)
+		if name == "ul" || name == "ol" {
+			return true
+		}
+		if name == "li" {
+			return false
+		}
+		if name == "#comment" && node.Data == ListEndCommentData {
+			return false
+		}
+
+		// If there is any text between two lists
+		// they are automatically not connected anymore.
+		if node.Type == html.TextNode {
+			return false
+		}
+
+		// - - - - //
+
+		if name == "hr" {
+			// A divider already seperates two lists...
+			return false
+		}
+
+		// TODO: RunContext.Render()
+		// -> get acess to keepRemoveMap
+
+		// TODO: look in the KeepRemoveMap?
+		// e.g. ul then script then ul
+
+		node = dom.GetNextNeighborNode(node)
+		continue
+	}
+	return false
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/list_items.go

@@ -0,0 +1,53 @@
+package domutils
+
+import (
+	"context"
+	"strings"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// MoveListItems moves non-"li" nodes into the previous "li" nodes.
+func MoveListItems(ctx context.Context, n *html.Node) {
+	if n.Type == html.ElementNode && (n.Data == "ol" || n.Data == "ul") {
+		var previousLi *html.Node
+
+		// Collect children to avoid modifying the slice while iterating.
+		children := dom.AllChildNodes(n)
+
+		for _, child := range children {
+			if child.Type == html.ElementNode && child.Data == "li" {
+				previousLi = child
+			} else if child.Type == html.TextNode && strings.TrimSpace(child.Data) == "" {
+				// Skip the node, probably just formatting of code
+			} else {
+				// We expect that inside an "ol"/"ul" there are *only* "li" nodes.
+				// But sometimes that is not the case...
+
+				if previousLi != nil {
+					// There is a previous "li" node,
+					// so we move this content into the other "li" node.
+					n.RemoveChild(child)
+
+					previousLi.AppendChild(child)
+				} else {
+					// There is no previous "li" node,
+					// so we wrap this node with it's own "li" node.
+
+					newNode := &html.Node{
+						Type:     html.ElementNode,
+						DataAtom: atom.Li,
+						Data:     "li",
+					}
+					previousLi = dom.WrapNode(child, newNode)
+				}
+			}
+		}
+	}
+
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		MoveListItems(ctx, c)
+	}
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/redundant.go

@@ -0,0 +1,29 @@
+package domutils
+
+import (
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func RemoveRedundant(doc *html.Node, matchFn func(*html.Node, *html.Node) bool) {
+	for _, node := range dom.AllNodes(doc) {
+		if hasSameTypeAncestor(node, matchFn) {
+			dom.UnwrapNode(node)
+		}
+	}
+
+}
+
+func hasSameTypeAncestor(n *html.Node, matchFn func(*html.Node, *html.Node) bool) bool {
+	if !matchFn(n, n) {
+		return false
+	}
+
+	for p := n.Parent; p != nil; p = p.Parent {
+		if matchFn(n, p) {
+			return true
+		}
+	}
+
+	return false
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/span.go

@@ -0,0 +1,50 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func isFakeSpan(node *html.Node) bool {
+	name := dom.NodeName(node)
+	if name != "span" {
+		return false
+	}
+
+	var containsBlockNode = false
+
+	var finder func(*html.Node)
+	finder = func(node *html.Node) {
+		name := dom.NodeName(node)
+		if dom.NameIsBlockNode(name) {
+			containsBlockNode = true
+			return
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+	finder(node)
+
+	return containsBlockNode
+}
+
+// RenameFakeSpans renames all "span" nodes to "div" if
+// any block element is found as a child.
+func RenameFakeSpans(ctx context.Context, doc *html.Node) {
+	var finder func(node *html.Node)
+	finder = func(node *html.Node) {
+		if isFakeSpan(node) {
+			node.Data = "div"
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+
+	finder(doc)
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/swap.go

@@ -0,0 +1,51 @@
+package domutils
+
+import (
+	"context"
+	"slices"
+	"strings"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func swapTagsOfNodes(node1, node2 *html.Node) {
+	if node1.Type != html.ElementNode || node2.Type != html.ElementNode {
+		panic("swap only works with element nodes")
+	}
+
+	tempDataAtom := node1.DataAtom
+	tempData := node1.Data
+	tempAttr := node1.Attr
+
+	node1.DataAtom = node2.DataAtom
+	node1.Data = node2.Data
+	node1.Attr = node2.Attr
+
+	node2.DataAtom = tempDataAtom
+	node2.Data = tempData
+	node2.Attr = tempAttr
+}
+
+func isEmptyText(node *html.Node) bool {
+	return node.Type == html.TextNode && strings.TrimSpace(node.Data) == ""
+}
+func SwapTags(ctx context.Context, doc *html.Node, isOuterNode, isInnerNode func(*html.Node) bool) {
+	var finder func(*html.Node)
+	finder = func(node *html.Node) {
+		if isOuterNode(node) {
+			childs := dom.AllChildNodes(node)
+			childs = slices.DeleteFunc(childs, isEmptyText)
+
+			if len(childs) == 1 && isInnerNode(childs[0]) {
+				swapTagsOfNodes(node, childs[0])
+				return
+			}
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+	finder(doc)
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/elem_backslash.go

@@ -0,0 +1,9 @@
+package escape
+
+func IsBackslash(chars []byte, index int) int {
+	if chars[index] != '\\' {
+		return -1
+	}
+
+	return 1
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/elem_code.go

@@ -0,0 +1,45 @@
+package escape
+
+func IsFencedCode(chars []byte, index int) int {
+	if chars[index] != '`' && chars[index] != '~' {
+		return -1
+	}
+
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == ' ' || chars[i] == placeholderByte {
+			continue
+		}
+		if chars[i] == '\n' {
+			break
+		}
+
+		return -1
+	}
+
+	count := 1
+	i := index + 1
+	for ; i < len(chars); i++ {
+		if chars[i] == placeholderByte {
+			continue
+		}
+		if chars[i] == '`' || chars[i] == '~' {
+			count++
+			continue
+		}
+
+		break
+	}
+	if count < 3 {
+		return -1
+	}
+
+	return i - index
+}
+
+func IsInlineCode(chars []byte, index int) int {
+	if chars[index] != '`' {
+		return -1
+	}
+
+	return 1
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/elem_divider.go

@@ -0,0 +1,47 @@
+package escape
+
+func IsDivider(chars []byte, index int) int {
+	if chars[index] != '-' && chars[index] != '_' && chars[index] != '*' {
+		return -1
+	}
+
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == '\n' {
+			break
+		}
+		if chars[i] == placeholderByte {
+			continue
+		}
+		if chars[i] == ' ' {
+			continue
+		}
+
+		return -1
+	}
+
+	count := 1
+	lastChar := len(chars)
+	for i := index + 1; i < len(chars); i++ {
+		if chars[i] == placeholderByte {
+			continue
+		}
+		if chars[i] == ' ' {
+			continue
+		}
+		if chars[i] == chars[index] {
+			count++
+			continue
+		}
+		if chars[i] == '\n' {
+			lastChar = i
+			break
+		}
+
+		return -1
+	}
+
+	if count >= 3 {
+		return lastChar - index
+	}
+	return -1
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/elem_header.go

@@ -0,0 +1,77 @@
+package escape
+
+func IsAtxHeader(chars []byte, index int) int {
+	if chars[index] != '#' {
+		return -1
+	}
+
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == '\n' {
+			break
+		}
+		if chars[i] == placeholderByte {
+			continue
+		}
+		if chars[i] == ' ' {
+			continue
+		}
+		return -1
+	}
+
+	nPoundSigns := 1
+	for i := index + 1; i < len(chars); i++ {
+		if chars[i] == '#' {
+			nPoundSigns++
+
+			if nPoundSigns > 6 {
+				return -1
+			}
+			continue
+		}
+
+		if chars[i] == placeholderByte {
+			continue
+		}
+		if chars[i] == ' ' || chars[i] == '\t' || chars[i] == '\n' || chars[i] == '\r' {
+			// TODO: fix calculation with placeholder (maybe own for loop construct?)
+			// Returns the count of # that we encountered
+			return i - index
+		}
+
+		return -1
+	}
+	return 1
+}
+
+func IsSetextHeader(chars []byte, index int) int {
+	if chars[index] != '=' && chars[index] != '-' {
+		return -1
+	}
+
+	var newlineCount int
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == placeholderByte || chars[i] == ' ' {
+			continue
+		}
+
+		if chars[i] == '\n' {
+			newlineCount++
+			continue
+		}
+
+		if newlineCount == 0 {
+			// Without any newlines, this character is on the same line
+			// as the delimiter. So the delimiter is inside a normal text...
+			return -1
+		} else if newlineCount == 1 {
+			// The heading content is on the line above the delimiter
+			// which qualifies for a setext heading...
+			return 1
+		} else {
+			return -1
+		}
+
+	}
+
+	return -1
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/elem_image.go

@@ -0,0 +1,36 @@
+package escape
+
+func IsImageOrLink(chars []byte, index int) int {
+	if chars[index] == '!' {
+		return isImageOrLinkStartExclamation(chars, index)
+	}
+	if chars[index] == '[' {
+		return isImageOrLinkStartBracket(chars, index)
+	}
+
+	return -1
+}
+
+func isImageOrLinkStartExclamation(chars []byte, index int) int {
+	nextIndex := index + 1
+	if nextIndex < len(chars) && chars[nextIndex] == '[' {
+		// It could be the start of an image
+		return 1
+	}
+
+	return -1
+}
+
+func isImageOrLinkStartBracket(chars []byte, index int) int {
+	for i := index + 1; i < len(chars); i++ {
+		if chars[i] == '\n' {
+			return -1
+		}
+
+		if chars[i] == ']' {
+			return 1
+		}
+	}
+
+	return -1
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/elem_italic_bold.go

@@ -0,0 +1,21 @@
+package escape
+
+import (
+	"unicode"
+)
+
+func IsItalicOrBold(chars []byte, index int) int {
+	if chars[index] != '*' && chars[index] != '_' {
+		return -1
+	}
+
+	next := getNextAsRune(chars, index)
+
+	nextIsWhitespace := unicode.IsSpace(next) || next == 0
+	if nextIsWhitespace {
+		// "not followed by Unicode whitespace"
+		return -1
+	}
+
+	return 1
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/elem_list.go

@@ -0,0 +1,67 @@
+package escape
+
+import (
+	"unicode"
+)
+
+func IsUnorderedList(chars []byte, index int) int {
+	if chars[index] != '-' && chars[index] != '*' && chars[index] != '+' {
+		return -1
+	}
+
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == '\n' {
+			break
+		}
+		if chars[i] == ' ' {
+			continue
+		}
+		if chars[i] == placeholderByte {
+			continue
+		}
+		return -1
+	}
+
+	next := getNext(chars, index)
+	if IsSpace(next) || next == 0 {
+		return 1
+	}
+
+	return -1
+}
+
+func IsOrderedList(chars []byte, index int) int {
+	if chars[index] != '.' && chars[index] != ')' {
+		return -1
+	}
+
+	// Directly before the dot needs to be a digit
+	prev := getPrevAsRune(chars, index)
+	if !unicode.IsDigit(prev) {
+		return -1
+	}
+
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == '\n' {
+			break
+		}
+		if chars[i] == ' ' {
+			continue
+		}
+		if chars[i] == placeholderByte {
+			continue
+		}
+		if IsDigit(chars[i]) {
+			continue
+		}
+
+		return -1
+	}
+
+	next := getNext(chars, index)
+	if IsSpace(next) || next == 0 {
+		return 1
+	}
+
+	return -1
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/elem_quote.go

@@ -0,0 +1,23 @@
+package escape
+
+func IsBlockQuote(chars []byte, index int) int {
+	if chars[index] != '>' {
+		return -1
+	}
+
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == '\n' {
+			break
+		}
+		if chars[i] == placeholderByte {
+			continue
+		}
+		if chars[i] == ' ' {
+			continue
+		}
+
+		return -1
+	}
+
+	return 1
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/replacer.go

@@ -0,0 +1,8 @@
+package escape
+
+import "github.com/JohannesKaufmann/html-to-markdown/v2/marker"
+
+var placeholderRune rune = marker.MarkerEscaping
+
+// IMPORTANT: Only internally we assume it is only byte
+var placeholderByte byte = marker.BytesMarkerEscaping[0]

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape/util.go

@@ -0,0 +1,70 @@
+package escape
+
+import "unicode/utf8"
+
+// TODO: move to markers package?
+
+func IsSpace(b byte) bool {
+	switch b {
+	case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0:
+		return true
+	}
+	return false
+}
+
+func IsDigit(b byte) bool {
+	switch b {
+	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+		return true
+	}
+	return false
+}
+
+func getPrev(chars []byte, index int) byte {
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == placeholderByte {
+			continue
+		}
+		return chars[i]
+	}
+	return 0
+}
+
+func getNext(chars []byte, index int) byte {
+	for i := index + 1; i < len(chars); i++ {
+		if chars[i] == placeholderByte {
+			continue
+		}
+		return chars[i]
+	}
+	return 0
+}
+
+func getPrevAsRune(chars []byte, index int) rune {
+	for i := index - 1; i >= 0; i-- {
+		if chars[i] == placeholderByte {
+			continue
+		}
+
+		r, _ := utf8.DecodeLastRune(chars[:i+1])
+
+		return r
+	}
+	return 0
+}
+func getNextAsRune(source []byte, index int) rune {
+	for i := index + 1; i < len(source); i++ {
+		if source[i] == placeholderByte {
+			continue
+		}
+
+		r, _ := utf8.DecodeRune(source[i:])
+		return r
+	}
+	return 0
+}
+
+// TODO: make public?
+func GetNextAsRune(source []byte, index int) rune {
+	return getNextAsRune(source, index)
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils/codefence.go

@@ -0,0 +1,61 @@
+package textutils
+
+import "strings"
+
+func CalculateCodeFenceOccurrences(fenceChar rune, content string) int {
+	var occurrences []int
+
+	var charsTogether int
+	for _, char := range content {
+		// We encountered a fence character, now count how many
+		// are directly afterwards
+		if char == fenceChar {
+			charsTogether++
+		} else if charsTogether != 0 {
+			occurrences = append(occurrences, charsTogether)
+			charsTogether = 0
+		}
+	}
+
+	// If the last element in the content was a fenceChar
+	if charsTogether != 0 {
+		occurrences = append(occurrences, charsTogether)
+	}
+
+	return findMax(occurrences)
+}
+
+// CalculateCodeFence can be passed the content of a code block and it returns
+// how many fence characters (` or ~) should be used.
+//
+// This is useful if the html content includes the same fence characters
+// for example ```
+// -> https://stackoverflow.com/a/49268657
+func CalculateCodeFence(fenceChar rune, content string) string {
+	repeat := CalculateCodeFenceOccurrences(fenceChar, content)
+
+	// The outer fence block always has to have
+	// at least one character more than any content inside
+	repeat++
+
+	// You have to have at least three fence characters
+	// to be recognized as a code block
+	if repeat < 3 {
+		repeat = 3
+	}
+
+	return strings.Repeat(string(fenceChar), repeat)
+}
+
+func findMax(a []int) (max int) {
+	for i, value := range a {
+		if i == 0 {
+			max = a[i]
+		}
+
+		if value > max {
+			max = value
+		}
+	}
+	return max
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils/collapse_code.go

@@ -0,0 +1,31 @@
+package textutils
+
+import (
+	"bytes"
+)
+
+func CollapseInlineCodeContent(content []byte) []byte {
+	// TODO: what about other characters like the reset char? Maybe unicode.IsSpace?
+	content = bytes.ReplaceAll(content, []byte{'\n'}, []byte{' '})
+	content = bytes.ReplaceAll(content, []byte{'\t'}, []byte{' '})
+
+	content = bytes.TrimSpace(content)
+
+	newChars := make([]byte, 0, len(content))
+
+	var count int
+	for _, char := range content {
+		if char == ' ' {
+			count++
+		} else {
+			count = 0
+		}
+
+		if count > 1 {
+			continue
+		}
+		newChars = append(newChars, char)
+	}
+
+	return newChars
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils/consecutive_newlines.go

@@ -0,0 +1,139 @@
+package textutils
+
+import (
+	"bytes"
+	"unicode/utf8"
+)
+
+func TrimUnnecessaryHardLineBreaks(content []byte) []byte {
+	content = bytes.ReplaceAll(content, []byte("  \n\n"), []byte("\n\n"))
+	content = bytes.ReplaceAll(content, []byte("  \n  \n"), []byte("\n\n"))
+	content = bytes.ReplaceAll(content, []byte("  \n \n"), []byte("\n\n"))
+	// out = bytes.ReplaceAll(out, []byte("\n  \n"), []byte("\n\n"))
+
+	return content
+}
+
+func TrimConsecutiveNewlines(input []byte) []byte {
+	var result []byte
+	newlineCount := 0
+	spaceBuffer := []byte{}
+
+	for i := 0; i < len(input); {
+		r, size := utf8.DecodeRune(input[i:])
+
+		if r == '\n' {
+			newlineCount++
+			if newlineCount <= 2 {
+				// Preserve up to 2 newlines, including preceding spaces
+				result = append(result, spaceBuffer...)
+				result = append(result, '\n')
+				spaceBuffer = spaceBuffer[:0] // Clear space buffer
+			} else {
+				// Skip additional newlines
+				spaceBuffer = spaceBuffer[:0] // Clear space buffer
+			}
+		} else if r == ' ' {
+			// Collect spaces into the space buffer
+			spaceBuffer = append(spaceBuffer, input[i:i+size]...)
+		} else {
+			// Reset newline count and append non-newline characters
+			newlineCount = 0
+			result = append(result, spaceBuffer...)
+			result = append(result, input[i:i+size]...)
+			spaceBuffer = spaceBuffer[:0] // Clear space buffer
+		}
+
+		i += size
+	}
+
+	// Append any trailing spaces
+	result = append(result, spaceBuffer...)
+
+	return result
+}
+
+/*
+func TrimConsecutiveNewlines(source []byte) []byte {
+	// Some performance optimizations:
+	// - If no replacement was done, we return the original slice and don't allocate.
+	// - We batch appends
+
+	var ret []byte
+
+	startNormal := 0
+	startMatch := -1
+
+	count := 0
+	// for i, b := range source {
+	for i := 0; i < len(source); i++ {
+		r, size := utf8.DecodeRune(source[i:])
+		_ = size
+
+		isNewline := r == '\n' // || r == marker.MarkerLineBreak
+		if isNewline {
+			count += 1
+		}
+
+		if startMatch == -1 && isNewline {
+			// Start of newlines
+			startMatch = i
+			i = i + size - 1
+			continue
+		} else if startMatch != -1 && isNewline {
+			// Middle of newlines
+			i = i + size - 1
+			continue
+		} else if startMatch != -1 {
+			// Character after the last newline character
+
+			if count > 2 {
+				if ret == nil {
+					ret = make([]byte, 0, len(source))
+				}
+
+				ret = append(ret, source[startNormal:startMatch]...)
+				ret = append(ret, '\n', '\n')
+				startNormal = i
+			}
+
+			startMatch = -1
+			count = 0
+		}
+	}
+
+	getStartEnd := func() (int, int, bool, bool) {
+		if startMatch == -1 && startNormal == 0 {
+			// a) no changes need to be done
+			return -1, -1, false, false
+		}
+
+		if count <= 2 {
+			// b) Only the normal characters still need to be added
+			return startNormal, len(source), true, false
+		}
+
+		// c) The match still needs to be replaced (and possible the previous normal characters be added)
+		return startNormal, startMatch, true, true
+	}
+
+	start, end, isKeepNeeded, isReplaceNeeded := getStartEnd()
+	if isKeepNeeded {
+		if ret == nil {
+			ret = make([]byte, 0, len(source))
+		}
+
+		ret = append(ret, source[start:end]...)
+		if isReplaceNeeded {
+			ret = append(ret, '\n', '\n')
+		}
+	}
+
+	if ret == nil {
+		// Hurray, we did not do any allocations with make()
+		// and instead just return the original slice.
+		return source
+	}
+	return ret
+}
+*/

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils/delimiter.go

@@ -0,0 +1,37 @@
+package textutils
+
+import (
+	"bytes"
+)
+
+// DelimiterForEveryLine puts the delimiter not just at the start and end of the string
+// but if the text is divided on multiple lines, puts the delimiters on every line with content.
+//
+// Otherwise the bold/italic delimiters won't be recognized if it contains new line characters.
+func DelimiterForEveryLine(text []byte, delimiter []byte) []byte {
+	var buf bytes.Buffer
+
+	lines := bytes.Split(text, []byte("\n"))
+	for i, line := range lines {
+		leftExtra, trimmed, rightExtra := SurroundingSpaces(line)
+
+		if trimmed == nil {
+			// For empty lines, we don't need a delimiter
+			buf.Write(leftExtra)
+			buf.Write(rightExtra)
+		} else {
+			buf.Write(leftExtra)
+			buf.Write(delimiter)
+			buf.Write(trimmed)
+			buf.Write(delimiter)
+			buf.Write(rightExtra)
+		}
+
+		// To join the lines again, add a newlines character
+		if i < len(lines)-1 {
+			buf.WriteRune('\n')
+		}
+	}
+
+	return buf.Bytes()
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils/escape_multiline.go

@@ -0,0 +1,56 @@
+package textutils
+
+import (
+	"bytes"
+	"unicode"
+)
+
+var (
+	doubleSpace = []byte{' ', ' '}
+
+	newlineBreak              = []byte{'\n'}
+	hardLineBreak             = []byte{' ', ' ', '\n'}
+	escapedNoContentLineBreak = []byte{'\\', '\n'}
+)
+
+// EscapeMultiLine deals with multiline content inside a link or a heading.
+func EscapeMultiLine(content []byte) []byte {
+	parts := bytes.Split(content, newlineBreak)
+	if len(parts) == 1 {
+		return content
+	}
+
+	output := make([]byte, 0, len(content))
+	for i := range parts {
+		trimmedLeft := bytes.TrimLeftFunc(parts[i], unicode.IsSpace)
+
+		if len(trimmedLeft) == 0 {
+			// A blank line would interrupt the link.
+			// So we need to escape the line
+			output = append(output, escapedNoContentLineBreak...)
+			continue
+		}
+
+		isLast := i == len(parts)-1
+		if isLast {
+			// For the last line we don't need to add any "\n" anymore
+			output = append(output, trimmedLeft...)
+			continue
+		}
+
+		// Now decide what ending we want:
+		if bytes.HasSuffix(trimmedLeft, doubleSpace) {
+			// We already have "  " so adding a "\n" is enough
+			output = append(output, trimmedLeft...)
+			output = append(output, newlineBreak...)
+			continue
+		} else {
+			// We *prefer* having a hard-line-break "  \n"
+			output = append(output, trimmedLeft...)
+			output = append(output, hardLineBreak...)
+			continue
+		}
+	}
+
+	return output
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils/prefix_lines.go

@@ -0,0 +1,16 @@
+package textutils
+
+func PrefixLines(source []byte, repl []byte) []byte {
+	newSlice := make([]byte, 0, len(source))
+
+	newSlice = append(newSlice, repl...)
+	for _, b := range source {
+		newSlice = append(newSlice, b)
+		if b == '\n' {
+			newSlice = append(newSlice, repl...)
+		}
+
+	}
+
+	return newSlice
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils/quote.go

@@ -0,0 +1,39 @@
+package textutils
+
+import "bytes"
+
+const (
+	DOUBLE_QUOTE = '"'
+	SINGLE_QUOTE = '\''
+)
+
+func SurroundBy(content []byte, chars []byte) []byte {
+	content = append(chars, content...)
+	content = append(content, chars...)
+	return content
+}
+func SurroundByQuotes(content []byte) []byte {
+	if len(content) == 0 {
+		return nil
+	}
+
+	containsDoubleQuote := bytes.ContainsRune(content, DOUBLE_QUOTE)
+	containsSingleQuote := bytes.ContainsRune(content, SINGLE_QUOTE)
+
+	if containsDoubleQuote && containsSingleQuote {
+		// Escape all quotes
+		content = bytes.ReplaceAll(content, []byte(`"`), []byte(`\"`))
+
+		// Surround the content by double quotes
+		return SurroundBy(content, []byte(`"`))
+	}
+	if containsDoubleQuote {
+		// Since it contains double quotes (but no single quotes)
+		// we can surround it by single quotes
+		return SurroundBy(content, []byte(`'`))
+	}
+
+	// It may contain single quotes, but definitely no double quotes,
+	// so we can safely surround it by double quotes.
+	return SurroundBy(content, []byte(`"`))
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils/surrounding_spaces.go

@@ -0,0 +1,58 @@
+/*
+
+The logic to handle whitespace around delimiters was initially developed
+in the fork from "anyproto" by Roman Khafizianov and Mikhail.
+
+The changes were then merged upstream by Johannes Kaufmann.
+
+https://github.com/anyproto/html-to-markdown
+https://github.com/JohannesKaufmann/html-to-markdown
+
+-----------
+
+MIT License
+
+Copyright (c) 2018 Johannes Kaufmann
+Copyright (c) 2020 Roman Khafizianov
+Copyright (c) 2023 Mikhail
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+package textutils
+
+import (
+	"bytes"
+	"unicode"
+)
+
+func SurroundingSpaces(content []byte) ([]byte, []byte, []byte) {
+	rightTrimmed := bytes.TrimRightFunc(content, func(r rune) bool {
+		return unicode.IsSpace(r)
+	})
+	rightExtra := content[len(rightTrimmed):]
+
+	trimmed := bytes.TrimLeftFunc(rightTrimmed, func(r rune) bool {
+		return unicode.IsSpace(r)
+	})
+	leftExtra := content[0 : len(rightTrimmed)-len(trimmed)]
+
+	return leftExtra, trimmed, rightExtra
+}