p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests
Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
This commit is contained in:
+138
@@ -0,0 +1,138 @@
|
||||
package base
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/collapse"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
type base struct{}
|
||||
|
||||
// NewBasePlugin registers a bunch of stuff that is not necessarily related to commonmark,
|
||||
// like removing nodes, trimming whitespace, collapsing whitespace, ...
|
||||
func NewBasePlugin() converter.Plugin {
|
||||
base := base{}
|
||||
return &base
|
||||
}
|
||||
|
||||
func (s *base) Name() string {
|
||||
return "base"
|
||||
}
|
||||
func (b *base) Init(conv *converter.Converter) error {
|
||||
conv.Register.TagType("#comment", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
conv.Register.TagType("head", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
conv.Register.TagType("script", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
conv.Register.TagType("style", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
conv.Register.TagType("link", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
conv.Register.TagType("meta", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
|
||||
conv.Register.TagType("iframe", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
conv.Register.TagType("noscript", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
|
||||
conv.Register.TagType("input", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
conv.Register.TagType("textarea", converter.TagTypeRemove, converter.PriorityStandard)
|
||||
|
||||
// "tr" is not in the `IsBlockNode` list,
|
||||
// but we want to treat is as a block anyway.
|
||||
// conv.Register.TagStrategy("tr", converter.StrategyMarkdownBlock, converter.PriorityStandard)
|
||||
// conv.Register.TagType("tr", converter.BlockTagType, converter.PriorityStandard)
|
||||
|
||||
conv.Register.PreRenderer(b.preRenderRemove, converter.PriorityEarly)
|
||||
// Note: The priority is low, so that collapse runs _after_ all the other functions
|
||||
conv.Register.PreRenderer(b.preRenderCollapse, converter.PriorityLate)
|
||||
|
||||
conv.Register.TextTransformer(b.handleTextTransform, converter.PriorityStandard)
|
||||
|
||||
conv.Register.PostRenderer(b.postRenderTrimContent, converter.PriorityStandard)
|
||||
conv.Register.PostRenderer(b.postRenderUnescapeContent, converter.PriorityStandard+20)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *base) preRenderRemove(ctx converter.Context, doc *html.Node) {
|
||||
var finder func(node *html.Node)
|
||||
finder = func(node *html.Node) {
|
||||
name := dom.NodeName(node)
|
||||
|
||||
if tagType, _ := ctx.GetTagType(name); tagType == converter.TagTypeRemove {
|
||||
dom.RemoveNode(node)
|
||||
return
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
// Because we are sometimes removing a node, this causes problems
|
||||
// with the for loop. Using `defer` is a cool trick!
|
||||
// https://gist.github.com/loopthrough/17da0f416054401fec355d338727c46e
|
||||
defer finder(child)
|
||||
}
|
||||
}
|
||||
finder(doc)
|
||||
|
||||
// - - - - - - - //
|
||||
|
||||
// After removing elements (see above) it can happen that we have
|
||||
// two #text nodes right next to each other. This would cause problems
|
||||
// with the collapse so we merge them together.
|
||||
domutils.MergeAdjacentTextNodes(doc)
|
||||
}
|
||||
|
||||
func (b *base) preRenderCollapse(ctx converter.Context, doc *html.Node) {
|
||||
collapse.Collapse(doc, &collapse.DomFuncs{
|
||||
IsBlockNode: func(node *html.Node) bool {
|
||||
tagName := dom.NodeName(node)
|
||||
tagType, ok := ctx.GetTagType(tagName)
|
||||
if ok {
|
||||
return tagType == converter.TagTypeBlock
|
||||
}
|
||||
|
||||
return dom.NameIsBlockNode(tagName)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
var characterEntityReplacer = strings.NewReplacer(
|
||||
// We are not using `html.EscapeString` because we
|
||||
// care about fewer characters
|
||||
"<", "<",
|
||||
">", ">",
|
||||
|
||||
// Note: We are not escaping "&" as "&" anymore.
|
||||
// In most cases the "&" is completely fine.
|
||||
// https://github.com/JohannesKaufmann/html-to-markdown/issues/178
|
||||
)
|
||||
|
||||
func (b *base) handleTextTransform(ctx converter.Context, content string) string {
|
||||
|
||||
// TODO: similar to UnEscapers also only escape if nessesary.
|
||||
// "<" only if not followed by space
|
||||
// "&" only if character entity
|
||||
content = characterEntityReplacer.Replace(content)
|
||||
|
||||
// TODO: reduce conversion between types
|
||||
content = string(ctx.EscapeContent([]byte(content)))
|
||||
|
||||
return content
|
||||
}
|
||||
|
||||
func (b *base) postRenderTrimContent(ctx converter.Context, result []byte) []byte {
|
||||
// Remove whitespace from the beginning & end
|
||||
result = bytes.TrimSpace(result)
|
||||
|
||||
// Remove too many newlines
|
||||
result = textutils.TrimConsecutiveNewlines(result)
|
||||
result = textutils.TrimUnnecessaryHardLineBreaks(result)
|
||||
|
||||
return result
|
||||
}
|
||||
func (b *base) postRenderUnescapeContent(ctx converter.Context, result []byte) []byte {
|
||||
result = ctx.UnEscapeContent(result)
|
||||
return result
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
package base
|
||||
|
||||
import (
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// RenderAsHTML will render the node as HTML using `html.Render()`
|
||||
// Newlines will be inserted depending on the configured `TagType`.
|
||||
//
|
||||
// As an example, you could do such a combination:
|
||||
//
|
||||
// "A text with <strong>bold</strong> and *italic* text"`
|
||||
func RenderAsHTML(ctx converter.Context, w converter.Writer, node *html.Node) converter.RenderStatus {
|
||||
tagName := dom.NodeName(node)
|
||||
tagType, _ := ctx.GetTagType(tagName)
|
||||
|
||||
if tagType == converter.TagTypeBlock {
|
||||
w.WriteString("\n\n")
|
||||
}
|
||||
_ = html.Render(w, node) // TODO: what to do with error?
|
||||
if tagType == converter.TagTypeBlock {
|
||||
w.WriteString("\n\n")
|
||||
}
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
|
||||
// RenderAsHTMLWrapper will render the node as HTML
|
||||
// and render the children as markdown.
|
||||
func RenderAsHTMLWrapper(ctx converter.Context, w converter.Writer, node *html.Node) converter.RenderStatus {
|
||||
name := dom.NodeName(node)
|
||||
|
||||
w.WriteString("<")
|
||||
w.WriteString(name)
|
||||
// TODO: also render the attributes?
|
||||
w.WriteString(">\n\n")
|
||||
|
||||
ctx.RenderChildNodes(ctx, w, node)
|
||||
|
||||
w.WriteString("\n\n</")
|
||||
w.WriteString(name)
|
||||
w.WriteString(">")
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
|
||||
// RenderAsPlaintextWrapper will keep the children of this node as markdown.
|
||||
func RenderAsPlaintextWrapper(ctx converter.Context, w converter.Writer, node *html.Node) converter.RenderStatus {
|
||||
tagName := dom.NodeName(node)
|
||||
tagType, _ := ctx.GetTagType(tagName)
|
||||
|
||||
if tagType == converter.TagTypeBlock {
|
||||
w.WriteString("\n\n")
|
||||
}
|
||||
ctx.RenderChildNodes(ctx, w, node)
|
||||
if tagType == converter.TagTypeBlock {
|
||||
w.WriteString("\n\n")
|
||||
}
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
Generated
Vendored
+206
@@ -0,0 +1,206 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/escape"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/marker"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
type commonmark struct {
|
||||
config
|
||||
}
|
||||
|
||||
type OptionFunc = func(config *config)
|
||||
|
||||
// _ or *
|
||||
//
|
||||
// default: *
|
||||
func WithEmDelimiter(delimiter string) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.EmDelimiter = delimiter
|
||||
}
|
||||
}
|
||||
|
||||
// ** or __
|
||||
//
|
||||
// default: **
|
||||
func WithStrongDelimiter(delimiter string) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.StrongDelimiter = delimiter
|
||||
}
|
||||
}
|
||||
|
||||
// Any Thematic break
|
||||
//
|
||||
// default: "* * *"
|
||||
func WithHorizontalRule(rule string) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.HorizontalRule = rule
|
||||
}
|
||||
}
|
||||
|
||||
// "-", "+", or "*"
|
||||
//
|
||||
// default: "-"
|
||||
func WithBulletListMarker(marker string) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.BulletListMarker = marker
|
||||
}
|
||||
}
|
||||
func WithListEndComment(enabled bool) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.DisableListEndComment = !enabled
|
||||
}
|
||||
}
|
||||
|
||||
// ``` or ~~~
|
||||
//
|
||||
// default: ```
|
||||
func WithCodeBlockFence(fence string) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.CodeBlockFence = fence
|
||||
}
|
||||
}
|
||||
|
||||
// "setext" or "atx"
|
||||
//
|
||||
// default: "atx"
|
||||
func WithHeadingStyle(style headingStyle) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.HeadingStyle = style
|
||||
}
|
||||
}
|
||||
|
||||
// WithLinkEmptyHrefBehavior configures how links with *empty hrefs* are rendered.
|
||||
// Take for example:
|
||||
//
|
||||
// <a href="">the link content</a>
|
||||
//
|
||||
// LinkBehaviorRenderAsLink would result in "[the link content]()"
|
||||
//
|
||||
// LinkBehaviorSkipLink would result in "the link content"
|
||||
func WithLinkEmptyHrefBehavior(behavior linkRenderingBehavior) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.LinkEmptyHrefBehavior = behavior
|
||||
}
|
||||
}
|
||||
|
||||
// WithLinkEmptyContentBehavior configures how links *without content* are rendered.
|
||||
// Take for example:
|
||||
//
|
||||
// <a href="/page"></a>
|
||||
//
|
||||
// LinkBehaviorRenderAsLink would result in "[](/page)"
|
||||
//
|
||||
// LinkBehaviorSkipLink would result in an empty string.
|
||||
func WithLinkEmptyContentBehavior(behavior linkRenderingBehavior) OptionFunc {
|
||||
return func(config *config) {
|
||||
config.LinkEmptyContentBehavior = behavior
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: allow changing the link style once the render logic is implemented
|
||||
//
|
||||
// "inlined" or "referenced_index" or "referenced_short"
|
||||
//
|
||||
// default: inlined
|
||||
// func WithLinkStyle(style linkStyle) OptionFunc {
|
||||
// return func(config *config) {
|
||||
// config.LinkStyle = style
|
||||
// }
|
||||
// }
|
||||
|
||||
// NewCommonmarkPlugin registers the markdown syntax of commonmark.
|
||||
func NewCommonmarkPlugin(opts ...OptionFunc) converter.Plugin {
|
||||
|
||||
cfg := &config{}
|
||||
for _, opt := range opts {
|
||||
opt(cfg)
|
||||
}
|
||||
|
||||
cm := commonmark{
|
||||
config: fillInDefaultConfig(cfg),
|
||||
}
|
||||
|
||||
return &cm
|
||||
}
|
||||
|
||||
func (s *commonmark) Name() string {
|
||||
return "commonmark"
|
||||
}
|
||||
func (cm *commonmark) Init(conv *converter.Converter) error {
|
||||
if err := validateConfig(&cm.config); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// - - - - - - - - //
|
||||
|
||||
conv.Register.PreRenderer(cm.handlePreRender, converter.PriorityStandard)
|
||||
|
||||
// Note: Should run after "collapse" & also after "remove"
|
||||
conv.Register.PreRenderer(func(ctx converter.Context, doc *html.Node) {
|
||||
if cm.DisableListEndComment {
|
||||
// Early return if the feature is unwanted
|
||||
return
|
||||
}
|
||||
|
||||
domutils.AddListEndComments(ctx, doc)
|
||||
}, converter.PriorityLate+100)
|
||||
|
||||
conv.Register.EscapedChar(
|
||||
'\\',
|
||||
'*', '_', '-', '+',
|
||||
'.', '>', '|',
|
||||
'$',
|
||||
'#', '=',
|
||||
'[', ']', '(', ')',
|
||||
'!',
|
||||
'~', '`', '"', '\'',
|
||||
)
|
||||
conv.Register.UnEscaper(escape.IsItalicOrBold, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsBlockQuote, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsAtxHeader, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsSetextHeader, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsDivider, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsOrderedList, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsUnorderedList, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsImageOrLink, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsFencedCode, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsInlineCode, converter.PriorityStandard)
|
||||
conv.Register.UnEscaper(escape.IsBackslash, converter.PriorityStandard)
|
||||
|
||||
conv.Register.Renderer(cm.handleRender, converter.PriorityStandard)
|
||||
|
||||
conv.Register.TextTransformer(cm.handleTextTransform, converter.PriorityLate)
|
||||
|
||||
conv.Register.PostRenderer(cm.handlePostRenderCodeBlockNewline, converter.PriorityLate)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cm commonmark) handlePostRenderCodeBlockNewline(ctx converter.Context, content []byte) []byte {
|
||||
return bytes.ReplaceAll(
|
||||
content,
|
||||
[]byte(string(marker.BytesMarkerCodeBlockNewline)),
|
||||
[]byte("\n"),
|
||||
)
|
||||
}
|
||||
|
||||
func (cm commonmark) handleTextTransform(ctx converter.Context, content string) string {
|
||||
|
||||
if isEnabled, ok := ctx.Value("is_inside_link").(bool); ok && isEnabled {
|
||||
content = strings.Replace(content, string(marker.MarkerEscaping)+`]`, `\]`, -1)
|
||||
}
|
||||
// if isEnabled, ok := ctx.Value("is_inside_heading").(bool); ok && isEnabled {
|
||||
// // The "#" character would be completely removed, if at the _end_
|
||||
// // of the heading content. So always escape it inside headings.
|
||||
// content = strings.Replace(content, string(marker.MarkerEscaping)+`#`, `\#`, -1)
|
||||
// }
|
||||
|
||||
return content
|
||||
}
|
||||
Generated
Vendored
+94
@@ -0,0 +1,94 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func nameIsBold(node *html.Node) bool {
|
||||
name := dom.NodeName(node)
|
||||
return name == "strong" || name == "b"
|
||||
}
|
||||
func nameIsItalic(node *html.Node) bool {
|
||||
name := dom.NodeName(node)
|
||||
return name == "em" || name == "i"
|
||||
}
|
||||
|
||||
func nameIsBoldOrItalic(node *html.Node) bool {
|
||||
return nameIsBold(node) || nameIsItalic(node)
|
||||
}
|
||||
func nameIsBothBoldOrItalic(a, b *html.Node) bool {
|
||||
if nameIsBold(a) && nameIsBold(b) {
|
||||
return true
|
||||
}
|
||||
if nameIsItalic(a) && nameIsItalic(b) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func nameIsPre(node *html.Node) bool {
|
||||
name := dom.NodeName(node)
|
||||
return name == "pre"
|
||||
}
|
||||
func nameIsInlineCode(node *html.Node) bool {
|
||||
name := dom.NodeName(node)
|
||||
return name == "code" || name == "var" || name == "samp" || name == "kbd" || name == "tt"
|
||||
}
|
||||
|
||||
func nameIsLink(node *html.Node) bool {
|
||||
return dom.NodeName(node) == "a"
|
||||
}
|
||||
|
||||
func nameIsBothLink(a, b *html.Node) bool {
|
||||
return dom.NodeName(a) == "a" && dom.NodeName(b) == "a"
|
||||
}
|
||||
|
||||
func nameIsHeading(node *html.Node) bool {
|
||||
name := dom.NodeName(node)
|
||||
|
||||
if name == "h1" || name == "h2" || name == "h3" || name == "h4" || name == "h5" || name == "h6" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// func nameIsBlockquote(node *html.Node) bool {
|
||||
// return dom.NodeName(node) == "blockquote"
|
||||
// }
|
||||
|
||||
func (c *commonmark) handlePreRender(ctx converter.Context, doc *html.Node) {
|
||||
domutils.RenameFakeSpans(ctx, doc)
|
||||
|
||||
// domutils.SplitUp(ctx, doc, nameIsBoldOrItalic, nameIsLink, atom.Span)
|
||||
|
||||
// domutils.SplitUp(ctx, doc, nameIsLink, nameIsHeading, atom.Div)
|
||||
// domutils.SplitUp(ctx, doc, nameIsLink, nameIsBlockquote, atom.Div)
|
||||
|
||||
// - - - Bold / Italic - - - //
|
||||
domutils.RemoveRedundant(doc, nameIsBothBoldOrItalic)
|
||||
domutils.MergeAdjacent(doc, nameIsBoldOrItalic)
|
||||
|
||||
// domutils.MovePunctuation(ctx, doc, nameIsBoldOrItalic)
|
||||
|
||||
// - - - Code - - - //
|
||||
domutils.RemoveEmptyCode(ctx, doc)
|
||||
domutils.SwapTags(ctx, doc, nameIsInlineCode, nameIsPre)
|
||||
domutils.MergeAdjacent(doc, nameIsInlineCode)
|
||||
|
||||
domutils.AddSpace(ctx, doc, nameIsBoldOrItalic, nameIsInlineCode)
|
||||
|
||||
// - - - Link - - - //
|
||||
domutils.RemoveRedundant(doc, nameIsBothLink)
|
||||
domutils.SwapTags(ctx, doc, nameIsBoldOrItalic, nameIsLink)
|
||||
|
||||
// - - - Heading - - - //
|
||||
domutils.SwapTags(ctx, doc, nameIsLink, nameIsHeading)
|
||||
domutils.LeafBlockAlternatives(ctx, doc)
|
||||
|
||||
// - - - List - - - //
|
||||
domutils.MoveListItems(ctx, doc)
|
||||
}
|
||||
Generated
Vendored
+47
@@ -0,0 +1,47 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func (c *commonmark) handleRender(ctx converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
name := dom.NodeName(n)
|
||||
|
||||
switch name {
|
||||
case "strong", "b",
|
||||
"em", "i":
|
||||
return c.renderBoldItalic(ctx, w, n)
|
||||
case "hr":
|
||||
return c.renderDivider(ctx, w, n)
|
||||
case "br":
|
||||
return c.renderBreak(ctx, w, n)
|
||||
case "ul", "ol":
|
||||
return c.renderListContainer(ctx, w, n)
|
||||
|
||||
case "pre":
|
||||
return c.renderBlockCode(ctx, w, n)
|
||||
case "code",
|
||||
"var", "samp", "kbd", "tt":
|
||||
return c.renderInlineCode(ctx, w, n)
|
||||
|
||||
case "blockquote":
|
||||
return c.renderBlockquote(ctx, w, n)
|
||||
|
||||
case "h1", "h2", "h3", "h4", "h5", "h6":
|
||||
return c.renderHeading(ctx, w, n)
|
||||
|
||||
case "img":
|
||||
return c.renderImage(ctx, w, n)
|
||||
|
||||
case "a":
|
||||
return c.renderLink(ctx, w, n)
|
||||
|
||||
case "#comment":
|
||||
return c.renderComment(ctx, w, n)
|
||||
}
|
||||
|
||||
return converter.RenderTryNext
|
||||
|
||||
}
|
||||
Generated
Vendored
+130
@@ -0,0 +1,130 @@
|
||||
package commonmark
|
||||
|
||||
type linkStyle string
|
||||
|
||||
const (
|
||||
// For example:
|
||||
//
|
||||
// [view more](/about.html)
|
||||
LinkStyleInlined linkStyle = "inlined"
|
||||
|
||||
LinkStyleReferencedIndex linkStyle = "referenced_index"
|
||||
LinkStyleReferencedShort linkStyle = "referenced_short"
|
||||
)
|
||||
|
||||
type headingStyle string
|
||||
|
||||
const (
|
||||
// HeadingStyleATX is the heading style of prefixing the heading with "#" signs indicating the level. For example:
|
||||
//
|
||||
// ## Heading
|
||||
HeadingStyleATX headingStyle = "atx"
|
||||
|
||||
// HeadingStyleSetext is the heading style of putting "=" or "-" on the followed line. For example:
|
||||
//
|
||||
// Heading
|
||||
// -------
|
||||
HeadingStyleSetext headingStyle = "setext"
|
||||
)
|
||||
|
||||
type linkRenderingBehavior string
|
||||
|
||||
const (
|
||||
// LinkBehaviorRender renders the element as a link
|
||||
LinkBehaviorRender linkRenderingBehavior = "render"
|
||||
// LinkBehaviorSkip skips link rendering and falls back to the other rules (e.g. paragraph)
|
||||
LinkBehaviorSkip linkRenderingBehavior = "skip"
|
||||
)
|
||||
|
||||
// config to customize the output. You can change stuff like
|
||||
// the character that is used for strong text.
|
||||
type config struct {
|
||||
// _ or *
|
||||
//
|
||||
// default: *
|
||||
EmDelimiter string
|
||||
|
||||
// ** or __
|
||||
//
|
||||
// default: **
|
||||
StrongDelimiter string
|
||||
|
||||
// Any Thematic break
|
||||
//
|
||||
// default: "* * *"
|
||||
HorizontalRule string
|
||||
|
||||
// "-", "+", or "*"
|
||||
//
|
||||
// default: "-"
|
||||
BulletListMarker string
|
||||
|
||||
DisableListEndComment bool
|
||||
|
||||
// "indented" or "fenced"
|
||||
//
|
||||
// default: "indented"
|
||||
// TODO: CodeBlockStyle string
|
||||
|
||||
// ``` or ~~~
|
||||
//
|
||||
// default: ```
|
||||
CodeBlockFence string
|
||||
|
||||
// "setext" or "atx"
|
||||
//
|
||||
// default: "atx"
|
||||
HeadingStyle headingStyle
|
||||
|
||||
// TODO: LineBreakStyle string "hard" or "soft"
|
||||
|
||||
// "inlined" or "referenced_index" or "referenced_short"
|
||||
//
|
||||
// default: inlined
|
||||
LinkStyle linkStyle
|
||||
|
||||
LinkEmptyHrefBehavior linkRenderingBehavior
|
||||
LinkEmptyContentBehavior linkRenderingBehavior
|
||||
}
|
||||
|
||||
func fillInDefaultConfig(cfg *config) config {
|
||||
if cfg.EmDelimiter == "" {
|
||||
// The new default is now "*" (instead of "_") as that works better inside words.
|
||||
cfg.EmDelimiter = "*"
|
||||
}
|
||||
if cfg.StrongDelimiter == "" {
|
||||
cfg.StrongDelimiter = "**"
|
||||
}
|
||||
|
||||
if cfg.HorizontalRule == "" {
|
||||
cfg.HorizontalRule = "* * *"
|
||||
}
|
||||
|
||||
if cfg.BulletListMarker == "" {
|
||||
cfg.BulletListMarker = "-"
|
||||
}
|
||||
|
||||
// TODO: also check for spelling mistakes in "indented"
|
||||
// if opt.CodeBlockStyle == "" {
|
||||
// opt.CodeBlockStyle = "indented"
|
||||
// }
|
||||
if cfg.CodeBlockFence == "" {
|
||||
cfg.CodeBlockFence = "```"
|
||||
}
|
||||
|
||||
if cfg.HeadingStyle == "" {
|
||||
cfg.HeadingStyle = "atx"
|
||||
}
|
||||
|
||||
if cfg.LinkEmptyHrefBehavior == "" {
|
||||
cfg.LinkEmptyHrefBehavior = LinkBehaviorRender
|
||||
}
|
||||
if cfg.LinkEmptyContentBehavior == "" {
|
||||
cfg.LinkEmptyContentBehavior = LinkBehaviorRender
|
||||
}
|
||||
if cfg.LinkStyle == "" {
|
||||
cfg.LinkStyle = LinkStyleInlined
|
||||
}
|
||||
|
||||
return *cfg
|
||||
}
|
||||
Generated
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func (c *commonmark) renderBlockquote(ctx converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
var buf bytes.Buffer
|
||||
ctx.RenderChildNodes(ctx, &buf, n)
|
||||
|
||||
content := buf.Bytes()
|
||||
content = bytes.TrimSpace(content)
|
||||
if content == nil {
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
|
||||
content = textutils.TrimConsecutiveNewlines(content)
|
||||
content = textutils.TrimUnnecessaryHardLineBreaks(content)
|
||||
content = textutils.PrefixLines(content, []byte{'>', ' '})
|
||||
|
||||
w.WriteRune('\n')
|
||||
w.WriteRune('\n')
|
||||
w.Write(content)
|
||||
w.WriteRune('\n')
|
||||
w.WriteRune('\n')
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
Generated
Vendored
+39
@@ -0,0 +1,39 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func (c commonmark) getDelimiter(n *html.Node) []byte {
|
||||
name := dom.NodeName(n)
|
||||
if name == "strong" || name == "b" {
|
||||
return []byte(c.StrongDelimiter)
|
||||
} else if name == "em" || name == "i" {
|
||||
return []byte(c.EmDelimiter)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
func (c commonmark) renderBoldItalic(ctx converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
var buf bytes.Buffer
|
||||
ctx.RenderChildNodes(ctx, &buf, n)
|
||||
|
||||
// Depending on the options & whether it is bold or italic there
|
||||
// is going to be a different delimiter.
|
||||
delimiter := c.getDelimiter(n)
|
||||
content := buf.Bytes()
|
||||
|
||||
// If there is a newline character between the start and end delimiter
|
||||
// the delimiters won't be recognized. Either we remove all newline characters
|
||||
// OR on _every_ line we put start & end delimiters.
|
||||
content = textutils.DelimiterForEveryLine(content, delimiter)
|
||||
|
||||
w.Write(content)
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
Generated
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func (c *commonmark) renderBreak(_ converter.Context, w converter.Writer, _ *html.Node) converter.RenderStatus {
|
||||
// Render a "hard line break"
|
||||
w.WriteString(" \n")
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
Generated
Vendored
+149
@@ -0,0 +1,149 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/marker"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func (c *commonmark) renderInlineCode(_ converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
// TODO: configure delimeter in options?
|
||||
fenceChar := '`'
|
||||
|
||||
codeContent, _ := getCodeWithoutTags(n)
|
||||
|
||||
// TODO: debug flag?
|
||||
if len(codeContent) == 0 {
|
||||
// fmt.Println("expected an empty inline code to be already removed")
|
||||
// panic("expected an empty inline code to be already removed")
|
||||
}
|
||||
// TODO: configurable function to decide if inline or block?
|
||||
if bytes.Contains(codeContent, []byte("\n")) {
|
||||
// fmt.Println("inline code contains newlines")
|
||||
// return c.renderBlockCode(ctx, w, n, render)
|
||||
}
|
||||
|
||||
if bytes.TrimSpace(codeContent) == nil {
|
||||
// No stripping occurs if the code span contains _only_ spaces:
|
||||
w.WriteRune(fenceChar)
|
||||
w.Write(codeContent)
|
||||
w.WriteRune(fenceChar)
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
|
||||
// Newlines in the text aren't great, since this is inline code and not a code block.
|
||||
// Newlines will be stripped anyway in the browser, but it won't be recognized as code
|
||||
// from the markdown parser when there is more than one newline.
|
||||
codeContent = textutils.CollapseInlineCodeContent(codeContent)
|
||||
|
||||
code := string(codeContent)
|
||||
|
||||
maxCount := textutils.CalculateCodeFenceOccurrences(fenceChar, code)
|
||||
maxCount++
|
||||
|
||||
fence := strings.Repeat(string(fenceChar), maxCount)
|
||||
|
||||
// Code contains a backtick as first character
|
||||
if strings.HasPrefix(code, "`") {
|
||||
code = " " + code
|
||||
}
|
||||
// Code contains a backtick as last character
|
||||
if strings.HasSuffix(code, "`") {
|
||||
code = code + " "
|
||||
}
|
||||
|
||||
w.WriteString(fence)
|
||||
w.WriteString(code)
|
||||
w.WriteString(fence)
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
func (c *commonmark) renderBlockCode(_ converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
code, infoString := getCodeWithoutTags(n)
|
||||
|
||||
if bytes.HasSuffix(code, []byte("\n")) {
|
||||
code = code[:len(code)-1]
|
||||
}
|
||||
|
||||
fenceChar, _ := utf8.DecodeRuneInString(c.CodeBlockFence)
|
||||
fence := textutils.CalculateCodeFence(fenceChar, string(code))
|
||||
|
||||
// We want to keep the original content inside the code block untouched.
|
||||
// Because multiple newlines would be trimmed, we temporarily replace it with another character.
|
||||
code = bytes.ReplaceAll(code, []byte("\n"), marker.BytesMarkerCodeBlockNewline)
|
||||
|
||||
w.WriteString("\n\n")
|
||||
w.WriteString(fence)
|
||||
w.WriteString(infoString)
|
||||
w.WriteRune('\n')
|
||||
w.Write(code)
|
||||
w.WriteRune('\n')
|
||||
w.WriteString(fence)
|
||||
w.WriteString("\n\n")
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
|
||||
func getCodeLanguage(n *html.Node) string {
|
||||
class := dom.GetAttributeOr(n, "class", "")
|
||||
|
||||
parts := strings.Split(class, " ")
|
||||
for _, part := range parts {
|
||||
if !strings.Contains(part, "language-") && !strings.Contains(part, "lang-") {
|
||||
continue
|
||||
}
|
||||
|
||||
part = strings.Replace(part, "language-", "", 1)
|
||||
part = strings.Replace(part, "lang-", "", 1)
|
||||
|
||||
return part
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
func getCodeWithoutTags(startNode *html.Node) ([]byte, string) {
|
||||
var buf bytes.Buffer
|
||||
var infoString string
|
||||
|
||||
var f func(*html.Node)
|
||||
f = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode && (n.Data == "code" || n.Data == "pre") {
|
||||
|
||||
// TODO: what if multiple elements have an info string?
|
||||
if infoString == "" {
|
||||
infoString = getCodeLanguage(n)
|
||||
}
|
||||
}
|
||||
|
||||
// - - - //
|
||||
|
||||
if n.Type == html.ElementNode && (n.Data == "style" || n.Data == "script" || n.Data == "textarea") {
|
||||
return
|
||||
}
|
||||
if n.Type == html.ElementNode && (n.Data == "br" || n.Data == "div") {
|
||||
buf.WriteString("\n")
|
||||
}
|
||||
|
||||
if n.Type == html.TextNode {
|
||||
// if strings.TrimSpace(n.Data) == "" && strings.Contains(n.Data, "\n") {
|
||||
// buf.WriteString("\n")
|
||||
// }
|
||||
buf.WriteString(n.Data)
|
||||
return
|
||||
}
|
||||
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
f(c)
|
||||
}
|
||||
}
|
||||
|
||||
f(startNode)
|
||||
|
||||
return buf.Bytes(), infoString
|
||||
}
|
||||
Generated
Vendored
+25
@@ -0,0 +1,25 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/domutils"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func (c *commonmark) renderComment(_ converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
|
||||
if n.Data == domutils.ListEndCommentData {
|
||||
// We definitely want to render the list end comments
|
||||
// that were just added
|
||||
w.WriteRune('\n')
|
||||
w.WriteRune('\n')
|
||||
_ = html.Render(w, n)
|
||||
w.WriteRune('\n')
|
||||
w.WriteRune('\n')
|
||||
return converter.RenderSuccess
|
||||
|
||||
}
|
||||
|
||||
// Fallback to the normal settings for comments
|
||||
return converter.RenderTryNext
|
||||
}
|
||||
Generated
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func (c *commonmark) renderDivider(_ converter.Context, w converter.Writer, _ *html.Node) converter.RenderStatus {
|
||||
|
||||
w.WriteString("\n\n")
|
||||
w.WriteString(c.HorizontalRule)
|
||||
w.WriteString("\n\n")
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
Generated
Vendored
+150
@@ -0,0 +1,150 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"regexp"
|
||||
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/marker"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// TODO: remove regex
|
||||
var multipleSpacesR = regexp.MustCompile(` +`)
|
||||
|
||||
func (r *commonmark) setextUnderline(level int, width int) []byte {
|
||||
line := "-"
|
||||
if level == 1 {
|
||||
line = "="
|
||||
}
|
||||
|
||||
return bytes.Repeat([]byte(line), width)
|
||||
}
|
||||
func (r *commonmark) atxPrefix(level int) []byte {
|
||||
return bytes.Repeat([]byte("#"), level)
|
||||
}
|
||||
|
||||
func getHeadingLevel(name string) int {
|
||||
switch name {
|
||||
case "h1":
|
||||
return 1
|
||||
case "h2":
|
||||
return 2
|
||||
case "h3":
|
||||
return 3
|
||||
case "h4":
|
||||
return 4
|
||||
case "h5":
|
||||
return 5
|
||||
case "h6":
|
||||
return 6
|
||||
default:
|
||||
return 6
|
||||
}
|
||||
}
|
||||
func runeCount(chars []rune) (count int) {
|
||||
for _, char := range chars {
|
||||
if char == marker.MarkerEscaping {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
}
|
||||
return
|
||||
}
|
||||
func getUnderlineWidth(content []byte, minVal int) int {
|
||||
var width int
|
||||
|
||||
parts := bytes.Split(content, []byte("\n"))
|
||||
for _, part := range parts {
|
||||
// Count how wide the line should be,
|
||||
// while using RuneCount to correctly count ä, ö, ...
|
||||
//
|
||||
// TODO: optimize function w := utf8.RuneCount(part)
|
||||
w := runeCount([]rune(string(part)))
|
||||
if w > width {
|
||||
width = w
|
||||
}
|
||||
}
|
||||
|
||||
// Technically the minimum value is only one character,
|
||||
// but one dash could easily trigger a heading.
|
||||
if width < minVal {
|
||||
return minVal
|
||||
}
|
||||
|
||||
return width
|
||||
}
|
||||
|
||||
func escapePoundSignAtEnd(s []byte) []byte {
|
||||
// -1 #
|
||||
// -2 placeholder
|
||||
// -3 maybe \
|
||||
|
||||
if s[len(s)-1] != '#' {
|
||||
// We don't have a # at the end,
|
||||
// so there is no work to do...
|
||||
return s
|
||||
}
|
||||
if len(s) >= 3 && s[len(s)-3] == '\\' {
|
||||
// It is already escaped,
|
||||
// so there is no work to do...
|
||||
return s
|
||||
}
|
||||
|
||||
// Because we have a # at the end,
|
||||
// we should manually force the escaping
|
||||
// by overriding the placeholder.
|
||||
s[len(s)-2] = '\\'
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func (c *commonmark) renderHeading(ctx converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
// ctx = context.WithValue(ctx, "is_inside_heading", true)
|
||||
|
||||
level := getHeadingLevel(dom.NodeName(n))
|
||||
|
||||
var buf bytes.Buffer
|
||||
ctx.RenderChildNodes(ctx, &buf, n)
|
||||
content := buf.Bytes()
|
||||
|
||||
if len(bytes.TrimSpace(content)) == 0 {
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
|
||||
if c.HeadingStyle == HeadingStyleSetext && level < 3 {
|
||||
// Note: We don't want to use `TrimUnnecessaryHardLineBreaks` here,
|
||||
// since `EscapeMultiLine` also takes care of newlines.
|
||||
content = textutils.TrimConsecutiveNewlines(content)
|
||||
content = textutils.EscapeMultiLine(content)
|
||||
|
||||
width := getUnderlineWidth(content, 3)
|
||||
underline := c.setextUnderline(level, width)
|
||||
|
||||
w.WriteString("\n\n")
|
||||
w.Write(content)
|
||||
w.WriteRune('\n')
|
||||
w.Write(underline)
|
||||
w.WriteString("\n\n")
|
||||
} else {
|
||||
content = bytes.ReplaceAll(content, []byte("\n"), []byte(" "))
|
||||
content = bytes.ReplaceAll(content, []byte("\r"), []byte(" "))
|
||||
// Replace multiple spaces by one space.
|
||||
content = multipleSpacesR.ReplaceAll(content, []byte(" "))
|
||||
|
||||
content = bytes.TrimSpace(content)
|
||||
|
||||
// A # sign at the end would be removed otherwise
|
||||
content = escapePoundSignAtEnd(content)
|
||||
|
||||
w.WriteString("\n\n")
|
||||
w.Write(c.atxPrefix(level))
|
||||
w.WriteRune(' ')
|
||||
w.Write(content)
|
||||
w.WriteString("\n\n")
|
||||
}
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
Generated
Vendored
+63
@@ -0,0 +1,63 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func escapeAlt(altString string) string {
|
||||
alt := []byte(altString)
|
||||
|
||||
var buf bytes.Buffer
|
||||
for i := range alt {
|
||||
if alt[i] == '[' || alt[i] == ']' {
|
||||
prevIndex := i - 1
|
||||
if prevIndex < 0 || alt[prevIndex] != '\\' {
|
||||
buf.WriteRune('\\')
|
||||
}
|
||||
}
|
||||
buf.WriteByte(alt[i])
|
||||
}
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func (c *commonmark) renderImage(ctx converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
src := dom.GetAttributeOr(n, "src", "")
|
||||
src = strings.TrimSpace(src)
|
||||
if src == "" {
|
||||
return converter.RenderTryNext
|
||||
}
|
||||
|
||||
src = ctx.AssembleAbsoluteURL(ctx, "img", src)
|
||||
|
||||
title := dom.GetAttributeOr(n, "title", "")
|
||||
title = strings.ReplaceAll(title, "\n", " ")
|
||||
|
||||
alt := dom.GetAttributeOr(n, "alt", "")
|
||||
alt = strings.ReplaceAll(alt, "\n", " ")
|
||||
|
||||
// The alt description will be placed between two square brackets `[alt]`
|
||||
// so make sure that those characters are escaped.
|
||||
alt = escapeAlt(alt)
|
||||
|
||||
w.WriteRune('!')
|
||||
w.WriteRune('[')
|
||||
w.WriteString(alt)
|
||||
w.WriteRune(']')
|
||||
w.WriteRune('(')
|
||||
w.WriteString(src)
|
||||
if title != "" {
|
||||
// The destination and title must be seperated by a space
|
||||
w.WriteRune(' ')
|
||||
w.Write(textutils.SurroundByQuotes([]byte(title)))
|
||||
}
|
||||
w.WriteRune(')')
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
Generated
Vendored
+103
@@ -0,0 +1,103 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// link in commonmark contains
|
||||
// - the link text (the visible text)
|
||||
// - a link destination (the URI that is the link destination)
|
||||
// - an optional link title
|
||||
type link struct {
|
||||
*html.Node
|
||||
|
||||
before []byte
|
||||
content []byte
|
||||
after []byte
|
||||
|
||||
href string
|
||||
title string
|
||||
}
|
||||
|
||||
func (c *commonmark) renderLinkInlined(w converter.Writer, l *link) converter.RenderStatus {
|
||||
|
||||
w.Write(l.before)
|
||||
w.WriteRune('[')
|
||||
w.Write(l.content)
|
||||
w.WriteRune(']')
|
||||
w.WriteRune('(')
|
||||
w.WriteString(l.href)
|
||||
if l.title != "" {
|
||||
// The destination and title must be separated by a space
|
||||
w.WriteRune(' ')
|
||||
w.Write(textutils.SurroundByQuotes([]byte(l.title)))
|
||||
}
|
||||
w.WriteRune(')')
|
||||
w.Write(l.after)
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
|
||||
func (c *commonmark) renderLink(ctx converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
ctx = ctx.WithValue("is_inside_link", true)
|
||||
|
||||
href := dom.GetAttributeOr(n, "href", "")
|
||||
|
||||
href = strings.TrimSpace(href)
|
||||
href = ctx.AssembleAbsoluteURL(ctx, "a", href)
|
||||
|
||||
if href == "" && c.config.LinkEmptyHrefBehavior == LinkBehaviorSkip {
|
||||
// There is *no href* for the link. Now we have two options:
|
||||
// Continue rendering as a link OR skip to let other renderers take over.
|
||||
return converter.RenderTryNext
|
||||
}
|
||||
|
||||
title := dom.GetAttributeOr(n, "title", "")
|
||||
title = strings.ReplaceAll(title, "\n", " ")
|
||||
|
||||
l := &link{
|
||||
Node: n,
|
||||
href: href,
|
||||
title: title,
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
ctx.RenderChildNodes(ctx, &buf, n)
|
||||
content := buf.Bytes()
|
||||
|
||||
if len(bytes.TrimSpace(content)) == 0 && c.config.LinkEmptyContentBehavior == LinkBehaviorSkip {
|
||||
// There is *no content* inside the link. Now we have two options:
|
||||
// Continue rendering as a link OR skip to let other renderers take over.
|
||||
return converter.RenderTryNext
|
||||
}
|
||||
|
||||
if l.href == "" {
|
||||
// A link without href is valid, like e.g. [text]()
|
||||
// But a title would make it invalid.
|
||||
l.title = ""
|
||||
}
|
||||
|
||||
leftExtra, trimmed, rightExtra := textutils.SurroundingSpaces(content)
|
||||
|
||||
// Note: We don't want to use `TrimUnnecessaryHardLineBreaks` here,
|
||||
// since `EscapeMultiLine` also takes care of newlines.
|
||||
trimmed = textutils.TrimConsecutiveNewlines(trimmed)
|
||||
trimmed = textutils.EscapeMultiLine(trimmed)
|
||||
|
||||
l.before = leftExtra
|
||||
l.content = trimmed
|
||||
l.after = rightExtra
|
||||
|
||||
switch c.LinkStyle {
|
||||
case LinkStyleInlined:
|
||||
return c.renderLinkInlined(w, l)
|
||||
default:
|
||||
return converter.RenderTryNext
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+111
@@ -0,0 +1,111 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/internal/textutils"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/marker"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func getStartAt(node *html.Node) int {
|
||||
startVal := dom.GetAttributeOr(node, "start", "1")
|
||||
startAt, err := strconv.Atoi(startVal)
|
||||
if err != nil {
|
||||
startAt = 1
|
||||
}
|
||||
|
||||
return startAt
|
||||
}
|
||||
|
||||
func (c commonmark) getPrefixFunc(n *html.Node, sliceLength int) func(int) string {
|
||||
startAt := getStartAt(n)
|
||||
|
||||
return func(sliceIndex int) string {
|
||||
if n.Data == "ul" {
|
||||
return c.BulletListMarker + " "
|
||||
}
|
||||
|
||||
currentIndex := startAt + sliceIndex
|
||||
lastIndex := startAt + sliceLength - 1
|
||||
maxLength := utf8.RuneCountInString(strconv.Itoa(lastIndex))
|
||||
|
||||
// Pad the numbers so that all prefix numbers in the list take up the same space
|
||||
// `%02d.` -> "01. "
|
||||
format := `%0` + strconv.Itoa(maxLength) + `d. `
|
||||
return fmt.Sprintf(format, currentIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func renderMultiLineListItem(w converter.Writer, content []byte, indentCount int) {
|
||||
lines := bytes.Split(content, []byte("\n"))
|
||||
indent := bytes.Repeat([]byte(" "), indentCount)
|
||||
|
||||
indentedCodeBlockNewline := append(marker.BytesMarkerCodeBlockNewline, indent...)
|
||||
|
||||
for i := range lines {
|
||||
// Add indent to code block newlines
|
||||
line := bytes.ReplaceAll(lines[i], marker.BytesMarkerCodeBlockNewline, indentedCodeBlockNewline)
|
||||
|
||||
if i != 0 {
|
||||
// The first line is already indented through the prefix,
|
||||
// all other lines need the correct amount of spaces.
|
||||
w.Write(indent)
|
||||
}
|
||||
w.Write(line)
|
||||
|
||||
if i < len(lines)-1 {
|
||||
w.WriteRune('\n')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c commonmark) renderListContainer(ctx converter.Context, w converter.Writer, n *html.Node) converter.RenderStatus {
|
||||
children := dom.AllChildNodes(n)
|
||||
items := make([][]byte, 0, len(children))
|
||||
|
||||
for _, child := range children {
|
||||
var buf bytes.Buffer
|
||||
ctx.RenderNodes(ctx, &buf, child)
|
||||
|
||||
content := buf.Bytes()
|
||||
content = bytes.TrimSpace(content)
|
||||
if content == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
items = append(items, content)
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
|
||||
getPrefix := c.getPrefixFunc(n, len(items))
|
||||
indentCount := utf8.RuneCountInString(getPrefix(0))
|
||||
|
||||
w.WriteString("\n\n")
|
||||
for i, item := range items {
|
||||
w.WriteString(getPrefix(i))
|
||||
|
||||
item = textutils.TrimConsecutiveNewlines(item)
|
||||
item = textutils.TrimUnnecessaryHardLineBreaks(item)
|
||||
item = ctx.UnEscapeContent(item)
|
||||
|
||||
// An item might have different lines that each
|
||||
// must be indented with the correct count of spaces.
|
||||
renderMultiLineListItem(w, item, indentCount)
|
||||
|
||||
if i < len(items)-1 {
|
||||
w.WriteRune('\n')
|
||||
}
|
||||
}
|
||||
w.WriteString("\n\n")
|
||||
|
||||
return converter.RenderSuccess
|
||||
}
|
||||
Generated
Vendored
+101
@@ -0,0 +1,101 @@
|
||||
package commonmark
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func contains(values []string, searchVal string) bool {
|
||||
for _, val := range values {
|
||||
if val == searchVal {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TODO: should this be with the commonmark package? Or more general?
|
||||
// TODO: Maybe make it an interface? And also have a GetPluginName function?
|
||||
type ValidateConfigError struct {
|
||||
Key string
|
||||
Value string
|
||||
|
||||
// By default is "Key:Value" but can be
|
||||
// overriden to e.g. "--key=value"
|
||||
KeyWithValue string
|
||||
|
||||
patternDescription string
|
||||
}
|
||||
|
||||
func (e *ValidateConfigError) setDefaultKeyWithValue() {
|
||||
e.KeyWithValue = fmt.Sprintf("%s:%q", e.Key, e.Value)
|
||||
}
|
||||
func (e *ValidateConfigError) Error() string {
|
||||
if e.KeyWithValue == "" {
|
||||
e.setDefaultKeyWithValue()
|
||||
}
|
||||
|
||||
return fmt.Sprintf("invalid value for %s must be %s", e.KeyWithValue, e.patternDescription)
|
||||
}
|
||||
|
||||
func validateConfig(cfg *config) error {
|
||||
if strings.Count(cfg.EmDelimiter, "_") != 1 && strings.Count(cfg.EmDelimiter, "*") != 1 {
|
||||
return &ValidateConfigError{
|
||||
Key: "EmDelimiter",
|
||||
Value: cfg.EmDelimiter,
|
||||
patternDescription: `exactly 1 character of "*" or "_"`,
|
||||
}
|
||||
}
|
||||
if strings.Count(cfg.StrongDelimiter, "_") != 2 && strings.Count(cfg.StrongDelimiter, "*") != 2 {
|
||||
return &ValidateConfigError{
|
||||
Key: "StrongDelimiter",
|
||||
Value: cfg.StrongDelimiter,
|
||||
patternDescription: `exactly 2 characters of "**" or "__"`,
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Count(cfg.HorizontalRule, "*") < 3 &&
|
||||
strings.Count(cfg.HorizontalRule, "_") < 3 &&
|
||||
strings.Count(cfg.HorizontalRule, "-") < 3 {
|
||||
return &ValidateConfigError{
|
||||
Key: "HorizontalRule",
|
||||
Value: cfg.HorizontalRule,
|
||||
patternDescription: `at least 3 characters of "*", "_" or "-"`,
|
||||
}
|
||||
}
|
||||
|
||||
if !contains([]string{"-", "+", "*"}, cfg.BulletListMarker) {
|
||||
return &ValidateConfigError{
|
||||
Key: "BulletListMarker",
|
||||
Value: cfg.BulletListMarker,
|
||||
patternDescription: `one of "-", "+" or "*"`,
|
||||
}
|
||||
}
|
||||
|
||||
if !contains([]string{"```", "~~~"}, cfg.CodeBlockFence) {
|
||||
return &ValidateConfigError{
|
||||
Key: "CodeBlockFence",
|
||||
Value: cfg.CodeBlockFence,
|
||||
patternDescription: "one of \"```\" or \"~~~\"",
|
||||
}
|
||||
}
|
||||
|
||||
if !contains([]string{"atx", "setext"}, string(cfg.HeadingStyle)) {
|
||||
return &ValidateConfigError{
|
||||
Key: "HeadingStyle",
|
||||
Value: string(cfg.HeadingStyle),
|
||||
patternDescription: `one of "atx" or "setext"`,
|
||||
}
|
||||
}
|
||||
|
||||
possibleLinkStyles := []string{string(LinkStyleInlined), string(LinkStyleReferencedIndex), string(LinkStyleReferencedShort)}
|
||||
if !contains(possibleLinkStyles, string(cfg.LinkStyle)) {
|
||||
return &ValidateConfigError{
|
||||
Key: "LinkStyle",
|
||||
Value: string(cfg.LinkStyle),
|
||||
patternDescription: `one of "inlined", "referenced_index" or "referenced_short"`,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user