p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
2026-09-22 21:54:01 -07:00
parent 6cb206fb27
commit 9444a35322
311 changed files with 238557 additions and 92 deletions
@@ -0,0 +1,142 @@
+package converter
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"slices"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+type convertOption struct {
+	domain  string
+	context context.Context
+}
+type ConvertOptionFunc func(o *convertOption)
+
+func WithContext(ctx context.Context) ConvertOptionFunc {
+	return func(o *convertOption) {
+		o.context = ctx
+	}
+}
+
+// WithDomain provides a base `domain` to the converter and
+// to the `AssembleAbsoluteURL` function.
+//
+// If a *relative* url is encountered (in an image or link) then the `domain` is used
+// to convert it to a *absolute* url.
+func WithDomain(domain string) ConvertOptionFunc {
+	return func(o *convertOption) {
+		o.domain = domain
+	}
+}
+
+func (conv *Converter) setError(err error) {
+	conv.m.Lock()
+	defer conv.m.Unlock()
+
+	conv.err = err
+}
+func (conv *Converter) getError() error {
+	conv.m.RLock()
+	defer conv.m.RUnlock()
+
+	return conv.err
+}
+
+var errNoRenderHandlers = errors.New(`no render handlers are registered. did you forget to register the "commonmark" and "base" plugins?`)
+var errBasePluginMissing = errors.New(`you registered the "commonmark" plugin but the "base" plugin is also required`)
+
+// ConvertNode converts a `*html.Node` to a markdown byte slice.
+//
+// If you have already parsed an HTML page using the `html.Parse()` function
+// from the "golang.org/x/net/html" package then you can pass this node
+// directly to the converter.
+func (conv *Converter) ConvertNode(doc *html.Node, opts ...ConvertOptionFunc) ([]byte, error) {
+
+	if err := conv.getError(); err != nil {
+		// There can be errors while calling `Init` on the plugins (e.g. validation errors).
+		// Now is the first opportunity where we can return that error.
+		return nil, err
+	}
+
+	conv.m.Lock()
+	option := &convertOption{}
+	for _, fn := range opts {
+		fn(option)
+	}
+	conv.m.Unlock()
+
+	// If there are no render handlers registered this is
+	// usually a user error - since people want the Commonmark Plugin in 99% of cases.
+	if len(conv.getRenderHandlers()) == 0 {
+		return nil, errNoRenderHandlers
+	}
+
+	containsCommonmark := slices.Contains(conv.registeredPlugins, "commonmark")
+	containsBase := slices.Contains(conv.registeredPlugins, "base")
+	if containsCommonmark && !containsBase {
+		return nil, errBasePluginMissing
+	}
+
+	// - - - - - - - - - - - - - - - - - - - //
+
+	state := newGlobalState()
+
+	if option.context == nil {
+		option.context = context.Background()
+	}
+	ctx := option.context
+	ctx = provideDomain(ctx, option.domain)
+	ctx = provideAssembleAbsoluteURL(ctx, defaultAssembleAbsoluteURL)
+	ctx = state.provideGlobalState(ctx)
+
+	customCtx := newConverterContext(ctx, conv)
+
+	// - - - - - - - - - - - - - - - - - - - //
+
+	// Pre-Render
+	for _, handler := range conv.getPreRenderHandlers() {
+		handler.Value(customCtx, doc)
+	}
+
+	// Render
+	var buf bytes.Buffer
+	conv.handleRenderNode(customCtx, &buf, doc)
+
+	// Post-Render
+	result := buf.Bytes()
+	for _, handler := range conv.getPostRenderHandlers() {
+		result = handler.Value(customCtx, result)
+	}
+
+	return result, nil
+}
+
+// ConvertReader converts the html from the reader to markdown.
+//
+// Under the hood `html.Parse()` is used to parse the HTML.
+func (conv *Converter) ConvertReader(r io.Reader, opts ...ConvertOptionFunc) ([]byte, error) {
+	doc, err := html.Parse(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return conv.ConvertNode(doc, opts...)
+}
+
+// ConvertString converts a html-string to a markdown-string.
+//
+// Under the hood `html.Parse()` is used to parse the HTML.
+func (conv *Converter) ConvertString(htmlInput string, opts ...ConvertOptionFunc) (string, error) {
+	r := strings.NewReader(htmlInput)
+	output, err := conv.ConvertReader(r, opts...)
+	if err != nil {
+		return "", err
+	}
+
+	return string(output), nil
+}
@@ -0,0 +1,72 @@
+package converter
+
+import "sync"
+
+type Converter struct {
+	m sync.RWMutex
+
+	err error
+
+	registeredPlugins []string
+
+	preRenderHandlers  prioritizedSlice[HandlePreRenderFunc]
+	renderHandlers     prioritizedSlice[HandleRenderFunc]
+	postRenderHandlers prioritizedSlice[HandlePostRenderFunc]
+
+	textTransformHandlers prioritizedSlice[HandleTextTransformFunc]
+
+	markdownChars    map[rune]interface{}
+	unEscapeHandlers prioritizedSlice[HandleUnEscapeFunc]
+
+	tagTypes map[string]prioritizedSlice[tagType]
+
+	escapeMode escapeMode
+
+	Register register
+}
+
+type converterOption = func(c *Converter) error
+
+func NewConverter(opts ...converterOption) *Converter {
+	conv := &Converter{
+		markdownChars: make(map[rune]interface{}),
+		tagTypes:      make(map[string]prioritizedSlice[tagType]),
+	}
+	conv.Register = register{conv}
+
+	for _, opt := range opts {
+		err := opt(conv)
+		if err != nil {
+			conv.setError(err)
+			break
+		}
+	}
+
+	return conv
+}
+
+type escapeMode string
+
+const (
+	EscapeModeDisabled escapeMode = "disabled"
+	EscapeModeSmart    escapeMode = "smart"
+)
+
+// WithEscapeMode changes the strictness of the "escaping".
+//
+// Some characters have a special meaning in markdown.
+// For example, the character "*" can be used for lists, emphasis and dividers.
+// By placing a backlash before that character (e.g. "\*") you can "escape" it.
+// Then the character will render as a raw "*" without the "markdown meaning" applied.
+//
+// Learn more in the documentation
+//
+//	"disabled" or "smart"
+//
+//	default: "smart"
+func WithEscapeMode(mode escapeMode) converterOption {
+	return func(c *Converter) error {
+		c.escapeMode = mode
+		return nil
+	}
+}
@@ -0,0 +1,190 @@
+package converter
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+// func GetValue[K string, V any](ctx context.Context, key K) V {
+// 	val, _ := ctx.Value(key).(V)
+// 	return val
+// }
+// func SetValue[K string, V any](ctx context.Context, key K, val V) context.Context {
+// 	return context.WithValue(ctx, key, val)
+// }
+
+type ctxKey string
+
+const (
+	ctxKeyAssembleAbsoluteURL ctxKey = "AssembleAbsoluteURL"
+	ctxKeyDomain              ctxKey = "Domain"
+
+	ctxKeySetState    ctxKey = "SetState"
+	ctxKeyUpdateState ctxKey = "UpdateState"
+	ctxKeyGetState    ctxKey = "GetState"
+)
+
+func provideDomain(ctx context.Context, domain string) context.Context {
+	return context.WithValue(ctx, ctxKeyDomain, domain)
+}
+func GetDomain(ctx context.Context) string {
+	domain, ok := ctx.Value(ctxKeyDomain).(string)
+	if !ok {
+		fmt.Println("[warning] value ctxKeyDomain is different")
+		return ""
+	}
+
+	return domain
+}
+
+// - - - - - - - - - - - - - - - - - - - - - //
+
+type AssembleAbsoluteURLFunc func(tagName string, rawURL string, domain string) string
+
+func assembleAbsoluteURL(ctx context.Context, tagName string, rawURL string) string {
+	domain := GetDomain(ctx)
+
+	// TODO: since this gets passed down from the converter, it doesn't have to provided from the ctx anymore
+	fn, ok := ctx.Value(ctxKeyAssembleAbsoluteURL).(AssembleAbsoluteURLFunc)
+	if !ok {
+		fmt.Println("[warning] func ctxKeyAssembleAbsoluteURL is different")
+		return ""
+	}
+
+	return fn(tagName, rawURL, domain)
+}
+
+func provideAssembleAbsoluteURL(ctx context.Context, fn AssembleAbsoluteURLFunc) context.Context {
+	return context.WithValue(ctx, ctxKeyAssembleAbsoluteURL, fn)
+}
+
+// - - - - - - - - - - - - - - - - - - - - - //
+
+type SetStateFunc func(key string, val any)
+type UpdateStateFunc func(key string, fn func(any) any)
+type GetStateFunc func(key string) any
+
+type globalState struct {
+	data map[string]any
+}
+
+func newGlobalState() *globalState {
+
+	return &globalState{
+		data: make(map[string]any),
+	}
+}
+
+func (s *globalState) setState(key string, val any) {
+	s.data[key] = val
+}
+func (s *globalState) updateState(key string, fn func(any) any) {
+	s.data[key] = fn(s.data[key])
+}
+func (s *globalState) getState(key string) any {
+	return s.data[key]
+}
+
+func (s *globalState) provideGlobalState(ctx context.Context) context.Context {
+
+	var setState SetStateFunc = s.setState
+	var updateState UpdateStateFunc = s.updateState
+	var getState GetStateFunc = s.getState
+
+	ctx = context.WithValue(ctx, ctxKeySetState, setState)
+	ctx = context.WithValue(ctx, ctxKeyUpdateState, updateState)
+	ctx = context.WithValue(ctx, ctxKeyGetState, getState)
+
+	return ctx
+}
+
+func GetState[V any](ctx context.Context, key string) V {
+	fn := ctx.Value(ctxKeyGetState).(GetStateFunc)
+
+	val, _ := fn(key).(V)
+
+	return val
+}
+
+func SetState[V any](ctx context.Context, key string, val V) {
+	fn := ctx.Value(ctxKeySetState).(SetStateFunc)
+
+	fn(key, val)
+}
+
+func UpdateState[V any](ctx context.Context, key string, fn func(V) V) {
+	updater := ctx.Value(ctxKeyUpdateState).(UpdateStateFunc)
+
+	updater(key, func(val any) any {
+		value, ok := val.(V)
+		if !ok && val != nil {
+			// TODO: slog?
+			fmt.Println("[warning] val is different than V in UpdateState")
+		}
+
+		return fn(value)
+	})
+}
+
+// - - - - - - //
+
+// Context extends the normal context.Context with some additional
+// methods useful for the process of converting.
+type Context interface {
+	context.Context
+
+	AssembleAbsoluteURL(ctx Context, tagName string, rawURL string) string
+
+	GetTagType(tagName string) (tagType, bool)
+
+	RenderNodes(ctx Context, w Writer, nodes ...*html.Node)
+	RenderChildNodes(ctx Context, w Writer, n *html.Node)
+
+	EscapeContent(content []byte) []byte
+	UnEscapeContent(content []byte) []byte
+
+	WithValue(key any, val any) Context
+}
+
+type converterContext struct {
+	context.Context
+	conv *Converter
+}
+
+func newConverterContext(ctx context.Context, conv *Converter) Context {
+	return &converterContext{
+		Context: ctx,
+		conv:    conv,
+	}
+}
+
+func (c *converterContext) AssembleAbsoluteURL(ctx Context, tagName string, rawURL string) string {
+	return assembleAbsoluteURL(ctx, tagName, rawURL)
+}
+
+func (c *converterContext) RenderNodes(ctx Context, w Writer, nodes ...*html.Node) {
+	c.conv.handleRenderNodes(ctx, w, nodes...)
+}
+func (c *converterContext) RenderChildNodes(ctx Context, w Writer, n *html.Node) {
+	c.conv.handleRenderNodes(ctx, w, dom.AllChildNodes(n)...)
+}
+
+func (c *converterContext) GetTagType(tagName string) (tagType, bool) {
+	return c.conv.getTagType(tagName)
+}
+func (c *converterContext) EscapeContent(content []byte) []byte {
+	return c.conv.escapeContent(content)
+}
+func (c *converterContext) UnEscapeContent(content []byte) []byte {
+	return c.conv.unEscapeContent(content)
+}
+
+func (c *converterContext) WithValue(key any, val any) Context {
+	return &converterContext{
+		Context: context.WithValue(c.Context, key, val),
+		conv:    c.conv,
+	}
+}
@@ -0,0 +1,89 @@
+package converter
+
+import (
+	"unicode/utf8"
+
+	"github.com/JohannesKaufmann/html-to-markdown/v2/marker"
+)
+
+const (
+	actionKeep   = iota
+	actionEscape = iota
+)
+
+// IMPORTANT: Only internally we assume it is only byte
+var placeholderByte byte = marker.BytesMarkerEscaping[0]
+
+func (conv *Converter) escapeContent(chars []byte) []byte {
+	if conv.escapeMode == EscapeModeDisabled {
+		return chars
+	}
+
+	newChars := make([]byte, 0, len(chars))
+	for index := 0; index < len(chars); index++ {
+		if chars[index] == '\u0000' {
+			// For security reasons, the Unicode character U+0000 must be replaced with the REPLACEMENT CHARACTER (U+FFFD).
+			newChars = append(newChars, []byte(string('\ufffd'))...)
+			continue
+		}
+
+		r, _ := utf8.DecodeRune(chars[index:])
+
+		isMarkdownChar := conv.checkIsEscapedChar(r)
+		if isMarkdownChar {
+			newChars = append(newChars, placeholderByte, chars[index])
+		} else {
+			newChars = append(newChars, chars[index])
+		}
+	}
+
+	return newChars
+}
+
+func (conv *Converter) unEscapeContent(chars []byte) []byte {
+	if conv.escapeMode == EscapeModeDisabled {
+		return chars
+	}
+
+	checkElements := func(index int) int {
+		for _, handler := range conv.getUnEscapeHandlers() {
+			if skip := handler.Value(chars, index); skip != -1 {
+				return skip
+			}
+		}
+
+		return -1
+	}
+
+	changes := make([]uint8, len(chars))
+	for index := 0; index < len(chars); index++ {
+
+		if chars[index] != placeholderByte {
+			continue
+		}
+		if index+1 >= len(chars) {
+			break
+		}
+
+		skip := checkElements(index + 1)
+		if skip == -1 {
+			continue
+		}
+		changes[index] = actionEscape
+		index += skip - 1
+	}
+
+	newChars := make([]byte, 0, len(chars))
+	for index, char := range chars {
+		if char != placeholderByte {
+			newChars = append(newChars, char)
+			continue
+		}
+
+		// What to do with this placeholder? Should we escape or not?
+		if changes[index] == actionEscape {
+			newChars = append(newChars, '\\')
+		}
+	}
+	return newChars
+}
@@ -0,0 +1,22 @@
+package converter
+
+// Plugin can be used to extends functionality beyond what
+// is offered by commonmark.
+type Plugin interface {
+	// The public name of the plugin, e.g. "strikethrough"
+	Name() string
+
+	// Init is called to initialize the plugin. It can be used to
+	// *validate* the arguments and *register* the rules.
+	Init(conv *Converter) error
+}
+
+// WithPlugins can be used to add additional functionality to the converter.
+func WithPlugins(plugins ...Plugin) converterOption {
+	return func(c *Converter) error {
+		for _, plugin := range plugins {
+			c.Register.Plugin(plugin)
+		}
+		return nil
+	}
+}
@@ -0,0 +1,33 @@
+package converter
+
+import "sort"
+
+const (
+	// PriorityEarly means that the handler will be run **early** in the process.
+	// To run it even earlier you need to subtract from this number.
+	PriorityEarly = 100
+
+	// PriorityStandard is for handlers that don't need to be run in a particular order.
+	PriorityStandard = 500
+
+	// PriorityLate means that the handler will be run **late** in the process.
+	// To run it even later you need to add to this number.
+	PriorityLate = 1000
+)
+
+type prioritizedValue[V any] struct {
+	Value    V
+	Priority int
+}
+
+type prioritizedSlice[V any] []prioritizedValue[V]
+
+func (s prioritizedSlice[V]) Sort() {
+	sort.Slice(s, func(i, j int) bool {
+		return s[i].Priority < s[j].Priority
+	})
+}
+
+func prioritized[V any](v V, priority int) prioritizedValue[V] {
+	return prioritizedValue[V]{v, priority}
+}
@@ -0,0 +1,222 @@
+package converter
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+type register struct {
+	conv *Converter
+}
+
+func (r *register) Plugin(plugin Plugin) {
+	pluginName := plugin.Name()
+	if pluginName == "" {
+		r.conv.setError(errors.New("the plugin has no name"))
+		return
+	}
+
+	r.conv.m.Lock()
+	r.conv.registeredPlugins = append(r.conv.registeredPlugins, pluginName)
+	r.conv.m.Unlock()
+
+	err := plugin.Init(r.conv)
+	if err != nil {
+		r.conv.setError(fmt.Errorf("error while initializing %q plugin: %w", pluginName, err))
+		return
+	}
+}
+
+// - - - - - - - - - - - - - Pre-Render - - - - - - - - - - - - - //
+
+type HandlePreRenderFunc func(ctx Context, doc *html.Node)
+
+func (r *register) PreRenderer(fn HandlePreRenderFunc, priority int) {
+	r.conv.m.Lock()
+	defer r.conv.m.Unlock()
+
+	handler := prioritized(fn, priority)
+	r.conv.preRenderHandlers = append(r.conv.preRenderHandlers, handler)
+}
+func (conv *Converter) getPreRenderHandlers() prioritizedSlice[HandlePreRenderFunc] {
+	conv.m.RLock()
+	defer conv.m.RUnlock()
+
+	handlers := make(prioritizedSlice[HandlePreRenderFunc], len(conv.preRenderHandlers))
+	copy(handlers, conv.preRenderHandlers)
+	handlers.Sort()
+
+	return handlers
+}
+
+// - - - - - - - - - - - - - Render - - - - - - - - - - - - - //
+
+// Writer is an interface that only conforms to the Write* methods of bytes.Buffer
+type Writer interface {
+	Write(p []byte) (n int, err error)
+	WriteByte(c byte) error
+	WriteRune(r rune) (n int, err error)
+	WriteString(s string) (n int, err error)
+}
+
+type HandleRenderFunc func(ctx Context, w Writer, n *html.Node) RenderStatus
+
+func (r *register) Renderer(fn HandleRenderFunc, priority int) {
+	r.conv.m.Lock()
+	defer r.conv.m.Unlock()
+
+	handler := prioritized(fn, priority)
+	r.conv.renderHandlers = append(r.conv.renderHandlers, handler)
+}
+
+// RendererFor registers a renderer for a specific tag (e.g. "div").
+// It is a small wrapper around `TagType()` and `Renderer()`.
+func (r *register) RendererFor(tagName string, tagType tagType, renderFn HandleRenderFunc, priority int) {
+
+	// 1. we add the "tagType" to the map
+	r.TagType(tagName, tagType, priority)
+
+	// 2. we register the render function
+	r.Renderer(func(ctx Context, w Writer, n *html.Node) RenderStatus {
+		if dom.NodeName(n) == tagName {
+			return renderFn(ctx, w, n)
+		}
+		return RenderTryNext
+	}, priority)
+}
+
+func (conv *Converter) getRenderHandlers() prioritizedSlice[HandleRenderFunc] {
+	conv.m.RLock()
+	defer conv.m.RUnlock()
+
+	handlers := make(prioritizedSlice[HandleRenderFunc], len(conv.renderHandlers))
+	copy(handlers, conv.renderHandlers)
+	handlers.Sort()
+
+	return handlers
+}
+
+// - - - - - - - - - - - - - Post Render - - - - - - - - - - - - - //
+
+type HandlePostRenderFunc func(ctx Context, content []byte) []byte
+
+func (r *register) PostRenderer(fn HandlePostRenderFunc, priority int) {
+	r.conv.m.Lock()
+	defer r.conv.m.Unlock()
+
+	handler := prioritized(fn, priority)
+	r.conv.postRenderHandlers = append(r.conv.postRenderHandlers, handler)
+}
+func (conv *Converter) getPostRenderHandlers() prioritizedSlice[HandlePostRenderFunc] {
+	conv.m.RLock()
+	defer conv.m.RUnlock()
+
+	handlers := make(prioritizedSlice[HandlePostRenderFunc], len(conv.postRenderHandlers))
+	copy(handlers, conv.postRenderHandlers)
+	handlers.Sort()
+
+	return handlers
+}
+
+// - - - - - - - - - - - - - Text - - - - - - - - - - - - - //
+
+type HandleTextTransformFunc func(ctx Context, content string) string
+
+func (r *register) TextTransformer(fn HandleTextTransformFunc, priority int) {
+	r.conv.m.Lock()
+	defer r.conv.m.Unlock()
+
+	handler := prioritized(fn, priority)
+	r.conv.textTransformHandlers = append(r.conv.textTransformHandlers, handler)
+}
+func (conv *Converter) getTextTransformHandlers() prioritizedSlice[HandleTextTransformFunc] {
+	conv.m.RLock()
+	defer conv.m.RUnlock()
+
+	handlers := make(prioritizedSlice[HandleTextTransformFunc], len(conv.textTransformHandlers))
+	copy(handlers, conv.textTransformHandlers)
+	handlers.Sort()
+
+	return handlers
+}
+
+// - - - - - - - - - - - - - Escaping - - - - - - - - - - - - - //
+
+func (r *register) EscapedChar(chars ...rune) {
+	r.conv.m.Lock()
+	defer r.conv.m.Unlock()
+
+	for _, char := range chars {
+		r.conv.markdownChars[char] = struct{}{}
+	}
+}
+func (conv *Converter) checkIsEscapedChar(r rune) bool {
+	conv.m.RLock()
+	defer conv.m.RUnlock()
+
+	_, ok := conv.markdownChars[r]
+	return ok
+}
+
+type HandleUnEscapeFunc func(chars []byte, index int) int
+
+func (r *register) UnEscaper(fn HandleUnEscapeFunc, priority int) {
+	r.conv.m.Lock()
+	defer r.conv.m.Unlock()
+
+	handler := prioritized(fn, priority)
+	r.conv.unEscapeHandlers = append(r.conv.unEscapeHandlers, handler)
+}
+func (conv *Converter) getUnEscapeHandlers() prioritizedSlice[HandleUnEscapeFunc] {
+	conv.m.RLock()
+	defer conv.m.RUnlock()
+
+	handlers := make(prioritizedSlice[HandleUnEscapeFunc], len(conv.unEscapeHandlers))
+	copy(handlers, conv.unEscapeHandlers)
+	handlers.Sort()
+
+	return handlers
+}
+
+// - - - - - - - - - - - - - Tag Type - - - - - - - - - - - - - //
+
+type tagType string
+
+const (
+	TagTypeBlock  tagType = "block"
+	TagTypeInline tagType = "inline"
+
+	// TagTypeRemove will remove that node in the _PreRender_ phase with a high priority.
+	TagTypeRemove tagType = "remove"
+)
+
+func (r *register) TagType(tagName string, tagType tagType, priority int) {
+	r.conv.m.Lock()
+	defer r.conv.m.Unlock()
+
+	val := prioritized(tagType, priority)
+	r.conv.tagTypes[tagName] = append(r.conv.tagTypes[tagName], val)
+}
+func (conv *Converter) getTagType(tagName string) (tagType, bool) {
+	conv.m.RLock()
+	defer conv.m.RUnlock()
+
+	types, ok := conv.tagTypes[tagName]
+	if !ok || len(types) == 0 {
+
+		if dom.NameIsBlockNode(tagName) {
+			return TagTypeBlock, true
+		} else if dom.NameIsInlineNode(tagName) {
+			return TagTypeInline, true
+		}
+		return "", false
+	}
+
+	types.Sort()
+	firstType := types[0].Value
+
+	return firstType, true
+}
@@ -0,0 +1,61 @@
+package converter
+
+import (
+	"github.com/JohannesKaufmann/dom"
+	"golang.org/x/net/html"
+)
+
+func (conv *Converter) handleRenderNodes(ctx Context, w Writer, nodes ...*html.Node) {
+	for _, node := range nodes {
+		conv.handleRenderNode(ctx, w, node)
+	}
+}
+
+func (conv *Converter) handleRenderNode(ctx Context, w Writer, node *html.Node) RenderStatus {
+	name := dom.NodeName(node)
+
+	// - - A: the #text node - - //
+	if name == "#text" {
+		return conv.handleRenderText(ctx, w, node)
+	}
+
+	// - - B: the render handlers - - //
+	for _, handler := range conv.getRenderHandlers() {
+		status := handler.Value(ctx, w, node)
+		if status == RenderSuccess {
+			return status
+		}
+	}
+
+	// - - C: the fallback - - //
+	// If nothing works we fallback to this:
+	return conv.handleRenderFallback(ctx, w, node)
+}
+
+func (conv *Converter) handleRenderFallback(ctx Context, w Writer, node *html.Node) RenderStatus {
+
+	tagName := dom.NodeName(node)
+	tagType, _ := ctx.GetTagType(tagName)
+
+	if tagType == TagTypeBlock {
+		w.WriteRune('\n')
+		w.WriteRune('\n')
+	}
+	ctx.RenderChildNodes(ctx, w, node)
+	if tagType == TagTypeBlock {
+		w.WriteRune('\n')
+		w.WriteRune('\n')
+	}
+
+	return RenderSuccess
+}
+func (conv *Converter) handleRenderText(ctx Context, w Writer, node *html.Node) RenderStatus {
+	content := node.Data
+
+	for _, handler := range conv.getTextTransformHandlers() {
+		content = handler.Value(ctx, content)
+	}
+
+	w.WriteString(content)
+	return RenderSuccess
+}
@@ -0,0 +1,8 @@
+package converter
+
+type RenderStatus int
+
+const (
+	RenderTryNext RenderStatus = iota
+	RenderSuccess
+)
@@ -0,0 +1,114 @@
+package converter
+
+import (
+	"net/url"
+	"strings"
+)
+
+var percentEncodingReplacer = strings.NewReplacer(
+	" ", "%20",
+	"[", "%5B",
+	"]", "%5D",
+	"(", "%28",
+	")", "%29",
+	"<", "%3C",
+	">", "%3E",
+)
+
+func parseBaseDomain(rawDomain string) *url.URL {
+	if rawDomain == "" {
+		return nil
+	}
+
+	u1, err := url.Parse(rawDomain)
+	if err == nil && u1.Host != "" {
+		// Yes, we got valid domain (probably with a http/https scheme)
+		return u1
+	}
+
+	u2, err := url.Parse("http://" + rawDomain)
+	if err == nil && u2.Host != "" {
+		// Yes, we got a valid domain (by choosing a fallback scheme)
+		return u2
+	}
+
+	return nil
+}
+func defaultAssembleAbsoluteURL(tagName string, rawURL string, domain string) string {
+	rawURL = strings.TrimSpace(rawURL)
+
+	if rawURL == "#" {
+		// Golangs url.Parse does not seem to distinguish between
+		// no fragment and an empty fragment.
+		return rawURL
+	}
+
+	// Increase the chance that the url will be parsed
+	rawURL = strings.ReplaceAll(rawURL, "\n", "%0A")
+	rawURL = strings.ReplaceAll(rawURL, "\t", "%09")
+
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		// We can't do anything with this url because it is invalid
+		return percentEncodingReplacer.Replace(rawURL)
+	}
+
+	if u.Scheme == "data" {
+		// This is a data uri (for example an inline base64 image)
+		return percentEncodingReplacer.Replace(rawURL)
+	}
+
+	// The default Query().Encode() encodes the query parameters "sorted by key".
+	// Instead we want to keep the original order, but still encode the parameters.
+	u.RawQuery = ParseAndEncodeQuery(u.RawQuery)
+
+	// For better compatibility (especially in regards to mailto links),
+	// instead of encoding a space with a "+" we use ""%20" to prevent
+	// e.g. the email reading "Hi+Johannes" instead of "Hi Johannes"
+	u.RawQuery = strings.ReplaceAll(u.RawQuery, "+", "%20")
+
+	if base := parseBaseDomain(domain); base != nil {
+		// If a "domain" is provided, we use that to convert relative links
+		// to absolute links.
+		u = base.ResolveReference(u)
+	}
+
+	return percentEncodingReplacer.Replace(u.String())
+}
+
+// - - - - //
+
+func decodeAndEncode(original string) string {
+	s, err := url.QueryUnescape(original)
+	if err != nil {
+		return original
+	}
+
+	return url.QueryEscape(s)
+}
+
+func ParseAndEncodeQuery(rawQuery string) string {
+	if rawQuery == "" {
+		return ""
+	}
+
+	rawParts := strings.Split(rawQuery, "&")
+	encodedParts := make([]string, len(rawParts))
+
+	for i, part := range rawParts {
+		splitted := strings.SplitN(part, "=", 2)
+
+		if len(splitted) == 1 {
+			// A: Just the key
+			encodedParts[i] = decodeAndEncode(splitted[0])
+		} else if splitted[1] == "" {
+			// B: The key and the equal sign
+			encodedParts[i] = decodeAndEncode(splitted[0]) + "="
+		} else {
+			// C: The key and the equal sign and the value
+			encodedParts[i] = decodeAndEncode(splitted[0]) + "=" + decodeAndEncode(splitted[1])
+		}
+	}
+
+	return strings.Join(encodedParts, "&")
+}