p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification
is ignored. No 502. keys add writes the secret only to --out. Release binary
has no -test-allow-prefix.

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/add_space.go

@@ -0,0 +1,66 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func getFirstChildNode(startNode *html.Node, matchFn func(n *html.Node) bool) *html.Node {
+	node := startNode.FirstChild
+	for node != nil {
+		name := dom.NodeName(node)
+		if name == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetNextNeighborNode(node)
+		} else if matchFn(node) {
+			return node
+		} else {
+			return nil
+		}
+	}
+
+	return nil
+}
+
+func getLastChildNode(startNode *html.Node, matchFn func(n *html.Node) bool) *html.Node {
+	node := startNode.LastChild
+	for node != nil {
+		name := dom.NodeName(node)
+		if name == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetPrevNeighborNode(node)
+		} else if matchFn(node) {
+			return node
+		} else {
+			return nil
+		}
+	}
+
+	return nil
+}
+func AddSpace(ctx context.Context, doc *html.Node, isOuterNode, isInnerNode func(*html.Node) bool) {
+	node := doc
+	for node != nil {
+		if isOuterNode(node) {
+			firstChild := getFirstChildNode(node, isInnerNode)
+			if firstChild != nil {
+				prev := getPrevTextNode(node)
+				if prev != nil {
+					prev.Data = prev.Data + " "
+				}
+			}
+
+			lastChild := getLastChildNode(node, isInnerNode)
+			if lastChild != nil {
+				next := getNextTextNode(node)
+				if next != nil {
+					next.Data = " " + next.Data
+				}
+			}
+		}
+
+		node = dom.GetNextNeighborElement(node)
+	}
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/adjacent.go

@@ -0,0 +1,75 @@
+package domutils
+
+import (
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func collectAdjacentNodes(node *html.Node, matchFn func(n *html.Node) bool) []*html.Node {
+	var collectedNodes []*html.Node
+
+	node = node.NextSibling
+	for node != nil {
+		name := dom.NodeName(node)
+		if name == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetNextNeighborNode(node)
+		} else if matchFn(node) {
+			collectedNodes = append(collectedNodes, node)
+			node = dom.GetNextNeighborNodeExcludingOwnChild(node)
+		} else {
+			// Return the collected nodes
+			return collectedNodes
+		}
+	}
+
+	return collectedNodes
+}
+
+func mergeChildren(destinationNode *html.Node, nodes ...*html.Node) {
+	for _, node := range nodes {
+		// We move all the children to the `destinationNode`.
+		children := dom.AllChildNodes(node)
+		for _, child := range children {
+			dom.RemoveNode(child)
+			destinationNode.AppendChild(child)
+		}
+		dom.RemoveNode(node)
+	}
+}
+
+func MergeAdjacent(doc *html.Node, matchFn func(*html.Node) bool) {
+	node := doc
+
+	for node != nil {
+		if matchFn(node) {
+			nextNodes := collectAdjacentNodes(node, matchFn)
+
+			mergeChildren(node, nextNodes...)
+		}
+
+		node = dom.GetNextNeighborElement(node)
+	}
+}
+
+// - - - - - - - - //
+
+func MergeAdjacentTextNodes(n *html.Node) {
+	if n == nil {
+		return
+	}
+
+	var prev *html.Node
+	for c := n.FirstChild; c != nil; {
+		next := c.NextSibling
+		if c.Type == html.TextNode && prev != nil && prev.Type == html.TextNode {
+			// Combine adjacent text nodes
+			prev.Data += c.Data
+			n.RemoveChild(c)
+		} else {
+			MergeAdjacentTextNodes(c)
+			prev = c
+		}
+		c = next
+	}
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/alternatives.go

@@ -0,0 +1,114 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+// TODO: make this configurable via the options???
+func getMarkdownStructure(name string) string {
+	switch name {
+	case "#document", "html", "head", "body",
+		"blockquote", "ul", "ol", "li":
+		// A container block can also contain other blocks.
+		return "container_block"
+
+	// Note: "p" would also be part of "leaf_block"
+	case "hr", "pre",
+		"h1", "h2", "h3", "h4", "h5", "h6":
+		// Leaf blocks can contain inline content
+		// but NOT other blocks.
+		return "leaf_block"
+
+	case "#text", "span", "code",
+		"b", "strong", "i", "em",
+		"a", "img", "br":
+		return "inline"
+
+	case "div", "p":
+		// Since these are just placing newlines,
+		// we dont categorize them.
+		return ""
+
+	default:
+		return ""
+	}
+}
+
+func headingAlternative(ctx context.Context, node *html.Node) {
+	node.Data = "strong"
+
+	newChild := &html.Node{
+		Type: html.ElementNode,
+		Data: "br",
+	}
+	node.Parent.InsertBefore(newChild, node.NextSibling)
+}
+func blockquoteAlternative(ctx context.Context, node *html.Node) {
+	newBefore := &html.Node{Type: html.TextNode, Data: ` "`}
+	node.Parent.InsertBefore(newBefore, node)
+
+	node.Data = "span"
+
+	newAfter := &html.Node{Type: html.TextNode, Data: `" `}
+	node.Parent.InsertBefore(newAfter, node.NextSibling)
+}
+func preAlternative(ctx context.Context, node *html.Node) {
+	node.Data = "code"
+}
+func hrAlternative(ctx context.Context, node *html.Node) {
+	dom.RemoveNode(node)
+}
+
+// TODO: make this configurable via the options?
+var alternatives = map[string]func(ctx context.Context, node *html.Node){
+	"h1":         headingAlternative,
+	"h2":         headingAlternative,
+	"h3":         headingAlternative,
+	"h4":         headingAlternative,
+	"h5":         headingAlternative,
+	"h6":         headingAlternative,
+	"blockquote": blockquoteAlternative,
+	"pre":        preAlternative,
+	"hr":         hrAlternative,
+}
+
+func LeafBlockAlternatives(ctx context.Context, doc *html.Node) {
+	var finder func(node *html.Node, isInsideLeafBlock bool, isInsideInline bool)
+	finder = func(node *html.Node, isInsideLeafBlock bool, isInsideInline bool) {
+		name := dom.NodeName(node)
+
+		structure := getMarkdownStructure(name)
+		if (structure == "container_block" || structure == "leaf_block") && (isInsideLeafBlock || isInsideInline) {
+			// A block inside an inline OR a block inside a leaf-block
+			// is not valid markdown so cannot be rendered.
+			//
+			// For example, you cannot place a blockquote inside a heading.
+			//
+			// Instead of this weird output (## Heading > My Quote)
+			// we try to find alternatives (## Heading "My Quote")
+			fn, ok := alternatives[name]
+			if ok {
+				fn(ctx, node)
+			} else {
+				node.Data = "span"
+			}
+		}
+
+		// - - - - - - - - - - - - - - - - - - - - - - //
+
+		if structure == "leaf_block" {
+			isInsideLeafBlock = true
+		}
+		if structure == "inline" {
+			isInsideInline = true
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			defer finder(child, isInsideLeafBlock, isInsideInline)
+		}
+	}
+	finder(doc, false, false)
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/domutils.go

@@ -0,0 +1,43 @@
+package domutils
+
+import (
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func getNextTextNode(startNode *html.Node) *html.Node {
+	node := dom.GetNextNeighborNodeExcludingOwnChild(startNode)
+
+	for node != nil {
+		if node.Type == html.TextNode {
+			return node
+		}
+
+		if dom.NodeName(node) == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetNextNeighborNode(node)
+			continue
+		}
+
+		return nil
+	}
+	return nil
+}
+func getPrevTextNode(startNode *html.Node) *html.Node {
+	node := dom.GetPrevNeighborNodeExcludingOwnChild(startNode)
+
+	for node != nil {
+		if node.Type == html.TextNode {
+			return node
+		}
+
+		if dom.NodeName(node) == "span" {
+			// A span has no special meaning. So we just skip it...
+			node = dom.GetPrevNeighborNode(node)
+			continue
+		}
+
+		return nil
+	}
+	return nil
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/empty_code.go

@@ -0,0 +1,42 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func hasTextChildNodes(startNode *html.Node) bool {
+	var found bool
+
+	var finder func(*html.Node)
+	finder = func(node *html.Node) {
+		if node.Type == html.TextNode && node.Data != "" {
+			found = true
+			return
+		}
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+	finder(startNode)
+
+	return found
+}
+
+func RemoveEmptyCode(ctx context.Context, doc *html.Node) {
+	node := doc
+	for node != nil {
+		if dom.NodeName(node) == "code" && !hasTextChildNodes(node) {
+			next := dom.GetNextNeighborNodeExcludingOwnChild(node)
+
+			dom.RemoveNode(node)
+
+			node = next
+			continue
+		}
+
+		node = dom.GetNextNeighborNode(node)
+	}
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/list_end_comment.go

@@ -0,0 +1,74 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+var ListEndCommentData = "THE END"
+
+func AddListEndComments(ctx context.Context, doc *html.Node) {
+	node := doc
+	for node != nil {
+		if nameIsList(node) && nextNameIsList(node) {
+			insertComment(node)
+		}
+
+		node = dom.GetNextNeighborElement(node)
+	}
+}
+
+func nameIsList(node *html.Node) bool {
+	name := dom.NodeName(node)
+	return name == "ul" || name == "ol"
+}
+
+func insertComment(listNode *html.Node) {
+	comment := &html.Node{
+		Type: html.CommentNode,
+		Data: ListEndCommentData,
+	}
+	listNode.Parent.InsertBefore(comment, listNode.NextSibling)
+}
+
+func nextNameIsList(startNode *html.Node) bool {
+	node := dom.GetNextNeighborNodeExcludingOwnChild(startNode)
+
+	for node != nil {
+		name := dom.NodeName(node)
+		if name == "ul" || name == "ol" {
+			return true
+		}
+		if name == "li" {
+			return false
+		}
+		if name == "#comment" && node.Data == ListEndCommentData {
+			return false
+		}
+
+		// If there is any text between two lists
+		// they are automatically not connected anymore.
+		if node.Type == html.TextNode {
+			return false
+		}
+
+		// - - - - //
+
+		if name == "hr" {
+			// A divider already seperates two lists...
+			return false
+		}
+
+		// TODO: RunContext.Render()
+		// -> get acess to keepRemoveMap
+
+		// TODO: look in the KeepRemoveMap?
+		// e.g. ul then script then ul
+
+		node = dom.GetNextNeighborNode(node)
+		continue
+	}
+	return false
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/list_items.go

@@ -0,0 +1,53 @@
+package domutils
+
+import (
+	"context"
+	"strings"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// MoveListItems moves non-"li" nodes into the previous "li" nodes.
+func MoveListItems(ctx context.Context, n *html.Node) {
+	if n.Type == html.ElementNode && (n.Data == "ol" || n.Data == "ul") {
+		var previousLi *html.Node
+
+		// Collect children to avoid modifying the slice while iterating.
+		children := dom.AllChildNodes(n)
+
+		for _, child := range children {
+			if child.Type == html.ElementNode && child.Data == "li" {
+				previousLi = child
+			} else if child.Type == html.TextNode && strings.TrimSpace(child.Data) == "" {
+				// Skip the node, probably just formatting of code
+			} else {
+				// We expect that inside an "ol"/"ul" there are *only* "li" nodes.
+				// But sometimes that is not the case...
+
+				if previousLi != nil {
+					// There is a previous "li" node,
+					// so we move this content into the other "li" node.
+					n.RemoveChild(child)
+
+					previousLi.AppendChild(child)
+				} else {
+					// There is no previous "li" node,
+					// so we wrap this node with it's own "li" node.
+
+					newNode := &html.Node{
+						Type:     html.ElementNode,
+						DataAtom: atom.Li,
+						Data:     "li",
+					}
+					previousLi = dom.WrapNode(child, newNode)
+				}
+			}
+		}
+	}
+
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		MoveListItems(ctx, c)
+	}
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/redundant.go

@@ -0,0 +1,29 @@
+package domutils
+
+import (
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func RemoveRedundant(doc *html.Node, matchFn func(*html.Node, *html.Node) bool) {
+	for _, node := range dom.AllNodes(doc) {
+		if hasSameTypeAncestor(node, matchFn) {
+			dom.UnwrapNode(node)
+		}
+	}
+
+}
+
+func hasSameTypeAncestor(n *html.Node, matchFn func(*html.Node, *html.Node) bool) bool {
+	if !matchFn(n, n) {
+		return false
+	}
+
+	for p := n.Parent; p != nil; p = p.Parent {
+		if matchFn(n, p) {
+			return true
+		}
+	}
+
+	return false
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/span.go

@@ -0,0 +1,50 @@
+package domutils
+
+import (
+	"context"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func isFakeSpan(node *html.Node) bool {
+	name := dom.NodeName(node)
+	if name != "span" {
+		return false
+	}
+
+	var containsBlockNode = false
+
+	var finder func(*html.Node)
+	finder = func(node *html.Node) {
+		name := dom.NodeName(node)
+		if dom.NameIsBlockNode(name) {
+			containsBlockNode = true
+			return
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+	finder(node)
+
+	return containsBlockNode
+}
+
+// RenameFakeSpans renames all "span" nodes to "div" if
+// any block element is found as a child.
+func RenameFakeSpans(ctx context.Context, doc *html.Node) {
+	var finder func(node *html.Node)
+	finder = func(node *html.Node) {
+		if isFakeSpan(node) {
+			node.Data = "div"
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+
+	finder(doc)
+}

vendor/github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils/swap.go

@@ -0,0 +1,51 @@
+package domutils
+
+import (
+	"context"
+	"slices"
+	"strings"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func swapTagsOfNodes(node1, node2 *html.Node) {
+	if node1.Type != html.ElementNode || node2.Type != html.ElementNode {
+		panic("swap only works with element nodes")
+	}
+
+	tempDataAtom := node1.DataAtom
+	tempData := node1.Data
+	tempAttr := node1.Attr
+
+	node1.DataAtom = node2.DataAtom
+	node1.Data = node2.Data
+	node1.Attr = node2.Attr
+
+	node2.DataAtom = tempDataAtom
+	node2.Data = tempData
+	node2.Attr = tempAttr
+}
+
+func isEmptyText(node *html.Node) bool {
+	return node.Type == html.TextNode && strings.TrimSpace(node.Data) == ""
+}
+func SwapTags(ctx context.Context, doc *html.Node, isOuterNode, isInnerNode func(*html.Node) bool) {
+	var finder func(*html.Node)
+	finder = func(node *html.Node) {
+		if isOuterNode(node) {
+			childs := dom.AllChildNodes(node)
+			childs = slices.DeleteFunc(childs, isEmptyText)
+
+			if len(childs) == 1 && isInnerNode(childs[0]) {
+				swapTagsOfNodes(node, childs[0])
+				return
+			}
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+	finder(doc)
+}