p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
2026-09-22 21:54:01 -07:00
parent 6cb206fb27
commit 9444a35322
311 changed files with 238557 additions and 92 deletions
@@ -0,0 +1,138 @@
+package base
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
+
+	"github.com/JohannesKaufmann/dom"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/collapse"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
+
+	"golang.org/x/net/html"
+)
+
+type base struct{}
+
+// NewBasePlugin registers a bunch of stuff that is not necessarily related to commonmark,
+// like removing nodes, trimming whitespace, collapsing whitespace, ...
+func NewBasePlugin() converter.Plugin {
+	base := base{}
+	return &base
+}
+
+func (s *base) Name() string {
+	return "base"
+}
+func (b *base) Init(conv *converter.Converter) error {
+	conv.Register.TagType("#comment", converter.TagTypeRemove, converter.PriorityStandard)
+	conv.Register.TagType("head", converter.TagTypeRemove, converter.PriorityStandard)
+	conv.Register.TagType("script", converter.TagTypeRemove, converter.PriorityStandard)
+	conv.Register.TagType("style", converter.TagTypeRemove, converter.PriorityStandard)
+	conv.Register.TagType("link", converter.TagTypeRemove, converter.PriorityStandard)
+	conv.Register.TagType("meta", converter.TagTypeRemove, converter.PriorityStandard)
+
+	conv.Register.TagType("iframe", converter.TagTypeRemove, converter.PriorityStandard)
+	conv.Register.TagType("noscript", converter.TagTypeRemove, converter.PriorityStandard)
+
+	conv.Register.TagType("input", converter.TagTypeRemove, converter.PriorityStandard)
+	conv.Register.TagType("textarea", converter.TagTypeRemove, converter.PriorityStandard)
+
+	// "tr" is not in the `IsBlockNode` list,
+	// but we want to treat is as a block anyway.
+	// conv.Register.TagStrategy("tr", converter.StrategyMarkdownBlock, converter.PriorityStandard)
+	// conv.Register.TagType("tr", converter.BlockTagType, converter.PriorityStandard)
+
+	conv.Register.PreRenderer(b.preRenderRemove, converter.PriorityEarly)
+	// Note: The priority is low, so that collapse runs _after_ all the other functions
+	conv.Register.PreRenderer(b.preRenderCollapse, converter.PriorityLate)
+
+	conv.Register.TextTransformer(b.handleTextTransform, converter.PriorityStandard)
+
+	conv.Register.PostRenderer(b.postRenderTrimContent, converter.PriorityStandard)
+	conv.Register.PostRenderer(b.postRenderUnescapeContent, converter.PriorityStandard+20)
+
+	return nil
+}
+
+func (b *base) preRenderRemove(ctx converter.Context, doc *html.Node) {
+	var finder func(node *html.Node)
+	finder = func(node *html.Node) {
+		name := dom.NodeName(node)
+
+		if tagType, _ := ctx.GetTagType(name); tagType == converter.TagTypeRemove {
+			dom.RemoveNode(node)
+			return
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			// Because we are sometimes removing a node, this causes problems
+			// with the for loop. Using `defer` is a cool trick!
+			// https://gist.github.com/loopthrough/17da0f416054401fec355d338727c46e
+			defer finder(child)
+		}
+	}
+	finder(doc)
+
+	// - - - - - - - //
+
+	// After removing elements (see above) it can happen that we have
+	// two #text nodes right next to each other. This would cause problems
+	// with the collapse so we merge them together.
+	domutils.MergeAdjacentTextNodes(doc)
+}
+
+func (b *base) preRenderCollapse(ctx converter.Context, doc *html.Node) {
+	collapse.Collapse(doc, &collapse.DomFuncs{
+		IsBlockNode: func(node *html.Node) bool {
+			tagName := dom.NodeName(node)
+			tagType, ok := ctx.GetTagType(tagName)
+			if ok {
+				return tagType == converter.TagTypeBlock
+			}
+
+			return dom.NameIsBlockNode(tagName)
+		},
+	})
+}
+
+var characterEntityReplacer = strings.NewReplacer(
+	// We are not using `html.EscapeString` because we
+	// care about fewer characters
+	"<", "&lt;",
+	">", "&gt;",
+
+	// Note: We are not escaping "&" as "&amp;" anymore.
+	// In most cases the "&" is completely fine.
+	// https://github.com/JohannesKaufmann/html-to-markdown/issues/178
+)
+
+func (b *base) handleTextTransform(ctx converter.Context, content string) string {
+
+	// TODO: similar to UnEscapers also only escape if nessesary.
+	//       "<" only if not followed by space
+	//       "&" only if character entity
+	content = characterEntityReplacer.Replace(content)
+
+	// TODO: reduce conversion between types
+	content = string(ctx.EscapeContent([]byte(content)))
+
+	return content
+}
+
+func (b *base) postRenderTrimContent(ctx converter.Context, result []byte) []byte {
+	// Remove whitespace from the beginning & end
+	result = bytes.TrimSpace(result)
+
+	// Remove too many newlines
+	result = textutils.TrimConsecutiveNewlines(result)
+	result = textutils.TrimUnnecessaryHardLineBreaks(result)
+
+	return result
+}
+func (b *base) postRenderUnescapeContent(ctx converter.Context, result []byte) []byte {
+	result = ctx.UnEscapeContent(result)
+	return result
+}
@@ -0,0 +1,62 @@
+package base
+
+import (
+	"github.com/JohannesKaufmann/dom"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
+	"golang.org/x/net/html"
+)
+
+// RenderAsHTML will render the node as HTML using `html.Render()`
+// Newlines will be inserted depending on the configured `TagType`.
+//
+// As an example, you could do such a combination:
+//
+//	"A text with <strong>bold</strong> and *italic* text"`
+func RenderAsHTML(ctx converter.Context, w converter.Writer, node *html.Node) converter.RenderStatus {
+	tagName := dom.NodeName(node)
+	tagType, _ := ctx.GetTagType(tagName)
+
+	if tagType == converter.TagTypeBlock {
+		w.WriteString("\n\n")
+	}
+	_ = html.Render(w, node) // TODO: what to do with error?
+	if tagType == converter.TagTypeBlock {
+		w.WriteString("\n\n")
+	}
+
+	return converter.RenderSuccess
+}
+
+// RenderAsHTMLWrapper will render the node as HTML
+// and render the children as markdown.
+func RenderAsHTMLWrapper(ctx converter.Context, w converter.Writer, node *html.Node) converter.RenderStatus {
+	name := dom.NodeName(node)
+
+	w.WriteString("<")
+	w.WriteString(name)
+	// TODO: also render the attributes?
+	w.WriteString(">\n\n")
+
+	ctx.RenderChildNodes(ctx, w, node)
+
+	w.WriteString("\n\n</")
+	w.WriteString(name)
+	w.WriteString(">")
+	return converter.RenderSuccess
+}
+
+// RenderAsPlaintextWrapper will keep the children of this node as markdown.
+func RenderAsPlaintextWrapper(ctx converter.Context, w converter.Writer, node *html.Node) converter.RenderStatus {
+	tagName := dom.NodeName(node)
+	tagType, _ := ctx.GetTagType(tagName)
+
+	if tagType == converter.TagTypeBlock {
+		w.WriteString("\n\n")
+	}
+	ctx.RenderChildNodes(ctx, w, node)
+	if tagType == converter.TagTypeBlock {
+		w.WriteString("\n\n")
+	}
+
+	return converter.RenderSuccess
+}