p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification
is ignored. No 502. keys add writes the secret only to --out. Release binary
has no -test-allow-prefix.

vendor/github.com/go-shiori/dom/.gitignore

@@ -0,0 +1,2 @@
+# Exclude config
+/.vscode

vendor/github.com/go-shiori/dom/.travis.yml

@@ -0,0 +1,11 @@
+language: go
+sudo: true
+matrix:
+  include:
+    - go: 1.x
+      env: LATEST=true
+
+script:
+  - go get -v ./...
+  - go vet $(go list ./... | grep -v /vendor/)
+  - go test -v -race ./...

vendor/github.com/go-shiori/dom/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019-present Radhi Fadlillah
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/go-shiori/dom/README.md

@@ -0,0 +1,22 @@
+DOM
+===
+
+[![GoDoc](https://godoc.org/github.com/go-shiori/dom?status.png)](https://godoc.org/github.com/go-shiori/dom)
+[![Travis CI](https://travis-ci.org/go-shiori/dom.svg?branch=master)](https://travis-ci.org/go-shiori/dom)
+[![Go Report Card](https://goreportcard.com/badge/github.com/go-shiori/dom)](https://goreportcard.com/report/github.com/go-shiori/dom)
+[![Donate PayPal](https://img.shields.io/static/v1?label=donate&message=PayPal&color=00457C&logo=paypal)](https://www.paypal.me/RadhiFadlillah)
+[![Donate Ko-fi](https://img.shields.io/static/v1?label=donate&message=Ko-fi&color=F16061&logo=ko-fi)](https://ko-fi.com/radhifadlillah)
+
+DOM is a Go package for manipulating [HTML](https://godoc.org/golang.org/x/net/html) node. The methods that exist in this package has similar name and purpose as the DOM manipulation methods in JS, which make it useful when porting code from JS to Go. Currently it used in [`warc`](https://github.com/go-shiori/warc) and [`go-readability`](https://github.com/go-shiori/go-readability).
+
+## Installation
+
+To install this package, just run `go get` :
+
+```
+go get -u -v github.com/go-shiori/dom
+```
+
+## Licenses
+
+DOM is distributed under [MIT license](https://choosealicense.com/licenses/mit/), which means you can use and modify it however you want. However, if you make an enhancement for it, if possible, please send a pull request. If you like this project, please consider donating to me either via [PayPal](https://www.paypal.me/RadhiFadlillah) or [Ko-Fi](https://ko-fi.com/radhifadlillah).

vendor/github.com/go-shiori/dom/dom.go

@@ -0,0 +1,622 @@
+package dom
+
+import (
+	"bytes"
+	"regexp"
+	"strings"
+
+	"github.com/andybalholm/cascadia"
+	"golang.org/x/net/html"
+)
+
+var (
+	rxPunctuation      = regexp.MustCompile(`\s+([.?!,;])\s*(\S*)`)
+	rxTempNewline      = regexp.MustCompile(`\s*\|\\/\|\s*`)
+	rxDisplayNone      = regexp.MustCompile(`(?i)display:\s*none`)
+	rxVisibilityHidden = regexp.MustCompile(`(?i)visibility:\s*(:?hidden|collapse)`)
+)
+
+// QuerySelectorAll returns array of document's elements that match
+// the specified group of selectors.
+func QuerySelectorAll(doc *html.Node, selectors string) []*html.Node {
+	matcher, err := cascadia.ParseGroup(selectors)
+	if err != nil {
+		return nil
+	}
+
+	return cascadia.QueryAll(doc, matcher)
+}
+
+// QuerySelector returns the first document's element that match
+// the specified group of selectors.
+func QuerySelector(doc *html.Node, selectors string) *html.Node {
+	matcher, err := cascadia.ParseGroup(selectors)
+	if err != nil {
+		return nil
+	}
+
+	return cascadia.Query(doc, matcher)
+}
+
+// GetElementByID returns a Node object representing the element whose id
+// property matches the specified string.
+func GetElementByID(doc *html.Node, id string) *html.Node {
+	if id == "" {
+		return nil
+	}
+
+	var results []*html.Node
+	var finder func(*html.Node)
+
+	finder = func(node *html.Node) {
+		nodeID := GetAttribute(node, "id")
+		nodeID = strings.TrimSpace(nodeID)
+
+		if node.Type == html.ElementNode && nodeID == id {
+			results = append(results, node)
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+
+	for child := doc.FirstChild; child != nil; child = child.NextSibling {
+		finder(child)
+
+		if len(results) > 0 {
+			return results[0]
+		}
+	}
+
+	return nil
+}
+
+// GetElementsByClassName returns an array of all child elements which
+// have all of the given class name(s).
+func GetElementsByClassName(doc *html.Node, classNames string) []*html.Node {
+	// Convert class name to map
+	classes := map[string]struct{}{}
+	for _, class := range strings.Fields(classNames) {
+		classes[class] = struct{}{}
+	}
+
+	nClasses := len(classes)
+	if nClasses == 0 {
+		return nil
+	}
+
+	// Create finder method
+	var results []*html.Node
+	var finder func(*html.Node)
+	allClassExist := func(node *html.Node) bool {
+		matchCount := 0
+		nodeClasses := GetAttribute(node, "class")
+		for _, nodeClass := range strings.Fields(nodeClasses) {
+			if _, exist := classes[nodeClass]; exist {
+				matchCount++
+			}
+		}
+
+		return matchCount == nClasses
+	}
+
+	finder = func(node *html.Node) {
+		if node.Type == html.ElementNode && allClassExist(node) {
+			results = append(results, node)
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+
+	// Check all nodes
+	for child := doc.FirstChild; child != nil; child = child.NextSibling {
+		finder(child)
+	}
+
+	return results
+}
+
+// GetElementsByTagName returns a collection of all elements in the document with
+// the specified tag name, as an array of Node object.
+// The special tag "*" will represents all elements.
+func GetElementsByTagName(doc *html.Node, tagName string) []*html.Node {
+	var results []*html.Node
+	var finder func(*html.Node)
+
+	finder = func(node *html.Node) {
+		if node.Type == html.ElementNode && (tagName == "*" || node.Data == tagName) {
+			results = append(results, node)
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+
+	for child := doc.FirstChild; child != nil; child = child.NextSibling {
+		finder(child)
+	}
+
+	return results
+}
+
+// CreateElement creates a new ElementNode with specified tag.
+func CreateElement(tagName string) *html.Node {
+	return &html.Node{
+		Type: html.ElementNode,
+		Data: tagName,
+	}
+}
+
+// CreateTextNode creates a new Text node.
+func CreateTextNode(data string) *html.Node {
+	return &html.Node{
+		Type: html.TextNode,
+		Data: data,
+	}
+}
+
+// TagName returns the tag name of a Node.
+// If it's not ElementNode, return empty string.
+func TagName(node *html.Node) string {
+	if node == nil {
+		return ""
+	}
+
+	if node.Type != html.ElementNode {
+		return ""
+	}
+
+	return node.Data
+}
+
+// GetAttribute returns the value of a specified attribute on
+// the element. If the given attribute does not exist, the value
+// returned will be an empty string.
+func GetAttribute(node *html.Node, attrName string) string {
+	for i := 0; i < len(node.Attr); i++ {
+		if node.Attr[i].Key == attrName {
+			return node.Attr[i].Val
+		}
+	}
+	return ""
+}
+
+// SetAttribute sets attribute for node. If attribute already exists,
+// it will be replaced.
+func SetAttribute(node *html.Node, attrName string, attrValue string) {
+	attrIdx := -1
+	for i := 0; i < len(node.Attr); i++ {
+		if node.Attr[i].Key == attrName {
+			attrIdx = i
+			break
+		}
+	}
+
+	if attrIdx >= 0 {
+		node.Attr[attrIdx].Val = attrValue
+	} else {
+		node.Attr = append(node.Attr, html.Attribute{
+			Key: attrName,
+			Val: attrValue,
+		})
+	}
+}
+
+// RemoveAttribute removes attribute with given name.
+func RemoveAttribute(node *html.Node, attrName string) {
+	attrIdx := -1
+	for i := 0; i < len(node.Attr); i++ {
+		if node.Attr[i].Key == attrName {
+			attrIdx = i
+			break
+		}
+	}
+
+	if attrIdx >= 0 {
+		a := node.Attr
+		a = append(a[:attrIdx], a[attrIdx+1:]...)
+		node.Attr = a
+	}
+}
+
+// HasAttribute returns a Boolean value indicating whether the
+// specified node has the specified attribute or not.
+func HasAttribute(node *html.Node, attrName string) bool {
+	for i := 0; i < len(node.Attr); i++ {
+		if node.Attr[i].Key == attrName {
+			return true
+		}
+	}
+	return false
+}
+
+// TextContent returns the text content of the specified node,
+// and all its descendants.
+func TextContent(node *html.Node) string {
+	var buffer bytes.Buffer
+	var finder func(*html.Node)
+
+	finder = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			buffer.WriteString(n.Data)
+		}
+
+		for child := n.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+
+	finder(node)
+	return buffer.String()
+}
+
+// InnerText in JS used to capture text from an element while excluding text from hidden
+// children. A child is hidden if it's computed width is 0, whether because its CSS (e.g
+// `display: none`, `visibility: hidden`, etc), or if the child has `hidden` attribute.
+// Since we can't compute stylesheet, we only look at `hidden` attribute and inline style.
+//
+// Besides excluding text from hidden children, difference between this function and
+// `TextContent` is the latter will skip <br> tag while this function will preserve
+// <br> as newline.
+func InnerText(node *html.Node) string {
+	var buffer bytes.Buffer
+	var finder func(*html.Node)
+
+	finder = func(n *html.Node) {
+		switch n.Type {
+		case html.TextNode:
+			buffer.WriteString(" " + n.Data + " ")
+
+		case html.ElementNode:
+			if n.Data == "br" {
+				buffer.WriteString(`|\/|`)
+				return
+			}
+
+			if HasAttribute(n, "hidden") {
+				return
+			}
+
+			styleAttr := GetAttribute(n, "style")
+			if rxDisplayNone.MatchString(styleAttr) || rxVisibilityHidden.MatchString(styleAttr) {
+				return
+			}
+		}
+
+		for child := n.FirstChild; child != nil; child = child.NextSibling {
+			finder(child)
+		}
+	}
+
+	finder(node)
+	text := buffer.String()
+	text = strings.Join(strings.Fields(text), " ")
+	text = rxPunctuation.ReplaceAllString(text, "$1 $2")
+	text = rxTempNewline.ReplaceAllString(text, "\n")
+	return text
+}
+
+// OuterHTML returns an HTML serialization of the element and its descendants.
+// The returned HTML value is escaped.
+func OuterHTML(node *html.Node) string {
+	if node == nil {
+		return ""
+	}
+
+	var buffer bytes.Buffer
+	err := html.Render(&buffer, node)
+	if err != nil {
+		return ""
+	}
+
+	return buffer.String()
+}
+
+// InnerHTML returns the HTML content (inner HTML) of an element.
+// The returned HTML value is escaped.
+func InnerHTML(node *html.Node) string {
+	var err error
+	var buffer bytes.Buffer
+
+	if node == nil {
+		return ""
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		err = html.Render(&buffer, child)
+		if err != nil {
+			return ""
+		}
+	}
+
+	return strings.TrimSpace(buffer.String())
+}
+
+// DocumentElement returns the Element that is the root element
+// of the document. Since we are working with HTML document,
+// the root will be <html> element for HTML documents).
+func DocumentElement(doc *html.Node) *html.Node {
+	if nodes := GetElementsByTagName(doc, "html"); len(nodes) > 0 {
+		return nodes[0]
+	}
+	return nil
+}
+
+// ID returns the value of the id attribute of the specified element.
+func ID(node *html.Node) string {
+	id := GetAttribute(node, "id")
+	id = strings.TrimSpace(id)
+	return id
+}
+
+// ClassName returns the value of the class attribute of
+// the specified element.
+func ClassName(node *html.Node) string {
+	className := GetAttribute(node, "class")
+	className = strings.TrimSpace(className)
+	className = strings.Join(strings.Fields(className), " ")
+	return className
+}
+
+// Children returns an HTMLCollection of the direct child elements of Node.
+func Children(node *html.Node) []*html.Node {
+	var children []*html.Node
+	if node == nil {
+		return nil
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if child.Type == html.ElementNode {
+			children = append(children, child)
+		}
+	}
+
+	return children
+}
+
+// ChildNodes returns list of a node's direct children.
+func ChildNodes(node *html.Node) []*html.Node {
+	var childNodes []*html.Node
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		childNodes = append(childNodes, child)
+	}
+	return childNodes
+}
+
+// FirstElementChild returns the object's first child Element,
+// or nil if there are no child elements.
+func FirstElementChild(node *html.Node) *html.Node {
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if child.Type == html.ElementNode {
+			return child
+		}
+	}
+	return nil
+}
+
+// PreviousElementSibling returns the the Element immediately prior
+// to the specified one in its parent's children list, or null if
+// the specified element is the first one in the list.
+func PreviousElementSibling(node *html.Node) *html.Node {
+	for sibling := node.PrevSibling; sibling != nil; sibling = sibling.PrevSibling {
+		if sibling.Type == html.ElementNode {
+			return sibling
+		}
+	}
+	return nil
+}
+
+// NextElementSibling returns the Element immediately following
+// the specified one in its parent's children list, or nil if the
+// specified Element is the last one in the list.
+func NextElementSibling(node *html.Node) *html.Node {
+	for sibling := node.NextSibling; sibling != nil; sibling = sibling.NextSibling {
+		if sibling.Type == html.ElementNode {
+			return sibling
+		}
+	}
+	return nil
+}
+
+// AppendChild adds a node to the end of the list of children of a
+// specified parent node. If the given child is a reference to an
+// existing node in the document, AppendChild() moves it from its
+// current position to the new position.
+func AppendChild(node *html.Node, child *html.Node) {
+	// Make sure node is not void
+	if !IsVoidElement(node) {
+		DetachChild(child)
+		node.AppendChild(child)
+	}
+}
+
+// PrependChild works like AppendChild() except it adds a node to the
+// beginning of the list of children of a specified parent node.
+func PrependChild(node *html.Node, child *html.Node) {
+	// Make sure node is not void
+	if !IsVoidElement(node) {
+		DetachChild(child)
+		if node.FirstChild != nil {
+			node.InsertBefore(child, node.FirstChild)
+		} else {
+			node.AppendChild(child)
+		}
+	}
+}
+
+// ReplaceChild replaces a child node within the given (parent) node.
+// If the new child is already exist in document, ReplaceChild() will move it
+// from its current position to replace old child. Returns both the new and old child.
+//
+// TODO: I'm note sure but I *think* there are some issues here. Check later I guess.
+func ReplaceChild(parent *html.Node, newChild *html.Node, oldChild *html.Node) (*html.Node, *html.Node) {
+	// Make sure parent is specified and not void
+	if parent == nil && !IsVoidElement(parent) {
+		return newChild, oldChild
+	}
+
+	// Make sure the specified parent IS the parent of the old child
+	if oldChild.Parent != parent {
+		return newChild, oldChild
+	}
+
+	// Detach the new child
+	DetachChild(newChild)
+	parent.InsertBefore(newChild, oldChild)
+	parent.RemoveChild(oldChild)
+	return newChild, oldChild
+}
+
+// IncludeNode determines if node is included inside nodeList.
+func IncludeNode(nodeList []*html.Node, node *html.Node) bool {
+	for i := 0; i < len(nodeList); i++ {
+		if nodeList[i] == node {
+			return true
+		}
+	}
+	return false
+}
+
+// Clone returns a clone of the node and (if specified) its children.
+// However, it will be detached from the original's parents and siblings.
+func Clone(src *html.Node, deep bool) *html.Node {
+	clone := &html.Node{
+		Type:     src.Type,
+		DataAtom: src.DataAtom,
+		Data:     src.Data,
+		Attr:     append([]html.Attribute{}, src.Attr...),
+	}
+
+	if deep {
+		for child := src.FirstChild; child != nil; child = child.NextSibling {
+			clone.AppendChild(Clone(child, deep))
+		}
+	}
+
+	return clone
+}
+
+// GetAllNodesWithTag is wrapper for GetElementsByTagName()
+// which allow to get several tags at once.
+func GetAllNodesWithTag(node *html.Node, tagNames ...string) []*html.Node {
+	var result []*html.Node
+	for i := 0; i < len(tagNames); i++ {
+		result = append(result, GetElementsByTagName(node, tagNames[i])...)
+	}
+	return result
+}
+
+// ForEachNode iterates over a NodeList and runs fn on each node.
+func ForEachNode(nodeList []*html.Node, fn func(*html.Node, int)) {
+	for i := 0; i < len(nodeList); i++ {
+		fn(nodeList[i], i)
+	}
+}
+
+// RemoveNodes iterates over a NodeList, calls `filterFn` for each node
+// and removes node if function returned `true`. If function is not
+// passed, removes all the nodes in node list.
+func RemoveNodes(nodeList []*html.Node, filterFn func(*html.Node) bool) {
+	for i := len(nodeList) - 1; i >= 0; i-- {
+		node := nodeList[i]
+		parentNode := node.Parent
+		if parentNode != nil && (filterFn == nil || filterFn(node)) {
+			parentNode.RemoveChild(node)
+		}
+	}
+}
+
+// SetTextContent sets the text content of the specified node.
+func SetTextContent(node *html.Node, text string) {
+	if IsVoidElement(node) {
+		return
+	}
+
+	child := node.FirstChild
+	for child != nil {
+		nextSibling := child.NextSibling
+		node.RemoveChild(child)
+		child = nextSibling
+	}
+
+	node.AppendChild(&html.Node{
+		Type: html.TextNode,
+		Data: text,
+	})
+}
+
+// SetInnerHTML sets inner HTML of the specified node.
+func SetInnerHTML(node *html.Node, rawHTML string) {
+	// Parse raw HTML
+	parsedHTML, err := html.Parse(strings.NewReader(rawHTML))
+	if err != nil || parsedHTML == nil {
+		return
+	}
+
+	// Remove node's current children
+	child := node.FirstChild
+	for child != nil {
+		nextSibling := child.NextSibling
+		node.RemoveChild(child)
+		child = nextSibling
+	}
+
+	// Put content of parsed HTML to the node
+	if body := QuerySelector(parsedHTML, "body"); body != nil {
+		bodyChild := body.FirstChild
+		for bodyChild != nil {
+			nextSibling := bodyChild.NextSibling
+			AppendChild(node, bodyChild)
+			bodyChild = nextSibling
+		}
+	}
+}
+
+// IsVoidElement check whether a node can have any contents or not.
+// Return true if element is void (can't have any children).
+func IsVoidElement(n *html.Node) bool {
+	// If it's not element, it's void
+	if n.Type != html.ElementNode {
+		return true
+	}
+
+	// Check tag name
+	switch n.Data {
+	case "area", "base", "br", "col", "embed", "hr",
+		"img", "input", "keygen", "link", "meta",
+		"param", "source", "track", "wbr":
+		return true
+	default:
+		return false
+	}
+}
+
+func DetachChild(child *html.Node) {
+	if child.Parent != nil || child.PrevSibling != nil || child.NextSibling != nil {
+		if child.Parent != nil {
+			if child.Parent.FirstChild == child {
+				child.Parent.FirstChild = child.NextSibling
+			}
+
+			if child.Parent.LastChild == child {
+				child.Parent.LastChild = child.PrevSibling
+			}
+		}
+
+		if child.PrevSibling != nil {
+			child.PrevSibling.NextSibling = child.NextSibling
+		}
+
+		if child.NextSibling != nil {
+			child.NextSibling.PrevSibling = child.PrevSibling
+		}
+
+		child.Parent = nil
+		child.PrevSibling = nil
+		child.NextSibling = nil
+	}
+}

vendor/github.com/go-shiori/dom/parser.go

@@ -0,0 +1,61 @@
+package dom
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+
+	"github.com/gogs/chardet"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/charset"
+	xunicode "golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/runes"
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
+)
+
+// FastParse parses html.Node from the specified reader without caring about
+// text encoding. It always assume that the input uses UTF-8 encoding.
+func FastParse(r io.Reader) (*html.Node, error) {
+	return html.Parse(r)
+}
+
+// Parse parses html.Node from the specified reader while converting the character
+// encoding into UTF-8. This function is useful to correctly parse web pages that
+// uses custom text encoding, e.g. web pages from Asian websites. However, since it
+// has to detect charset before parsing, this function is quite slow and expensive
+// so if you sure the reader uses valid UTF-8, just use FastParse.
+func Parse(r io.Reader) (*html.Node, error) {
+	// Split the reader using tee
+	content, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// Detect page encoding
+	res, err := chardet.NewHtmlDetector().DetectBest(content)
+	if err != nil {
+		return nil, err
+	}
+
+	pageEncoding, _ := charset.Lookup(res.Charset)
+	if pageEncoding == nil {
+		pageEncoding = xunicode.UTF8
+	}
+
+	// Parse HTML using the page encoding
+	r = bytes.NewReader(content)
+	r = transform.NewReader(r, pageEncoding.NewDecoder())
+	r = normalizeTextEncoding(r)
+	return html.Parse(r)
+}
+
+// normalizeTextEncoding convert text encoding from NFD to NFC.
+// It also remove soft hyphen since apparently it's useless in web.
+// See: https://web.archive.org/web/19990117011731/http://www.hut.fi/~jkorpela/shy.html
+func normalizeTextEncoding(r io.Reader) io.Reader {
+	fnSoftHyphen := func(r rune) bool { return r == '\u00AD' }
+	softHyphenSet := runes.Predicate(fnSoftHyphen)
+	transformer := transform.Chain(norm.NFD, runes.Remove(softHyphenSet), norm.NFC)
+	return transform.NewReader(r, transformer)
+}

vendor/github.com/go-shiori/go-readability/.gitattributes

@@ -0,0 +1 @@
+test-pages/* linguist-vendored

vendor/github.com/go-shiori/go-readability/.gitignore

@@ -0,0 +1,6 @@
+.vscode/
+js/*
+
+# Hidden script for development
+scripts/internal/
+.DS_Store

vendor/github.com/go-shiori/go-readability/.travis.yml

@@ -0,0 +1,16 @@
+language: go
+sudo: true
+matrix:
+  include:
+    - go: 1.x
+      env: LATEST=true
+    - go: "1.13"
+
+install:
+  - go get github.com/go-shiori/go-readability
+
+script:
+  - go get -t -v ./...
+  - diff -u <(echo -n) <(gofmt -d .)
+  - go vet $(go list ./... | grep -v /vendor/)
+  - go test -v -race ./...

vendor/github.com/go-shiori/go-readability/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Radhi Fadlillah
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/go-shiori/go-readability/Makefile

@@ -0,0 +1,10 @@
+generate:
+	@for name in internal/re2go/*.re; do \
+		RE_IN=$$name; \
+		RE_OUT=$$(echo $$name | sed 's/\.re/.go/'); \
+		re2go -W -F --input-encoding utf8 --utf8 --no-generation-date -i $$RE_IN -o $$RE_OUT; \
+		gofmt -w $$RE_OUT; \
+	done
+
+test: generate
+	go test -timeout 30s ./...

vendor/github.com/go-shiori/go-readability/README.md

@@ -0,0 +1,182 @@
+# Go-Readability [![Go Reference][go-ref-badge]][go-ref] [![PayPal][paypal-badge]][paypal] [![Ko-fi][kofi-badge]][kofi]
+
+Go-Readability is a Go package that find the main readable content and the metadata from a HTML page. It works by removing clutter like buttons, ads, background images, script, etc.
+
+This package is based from [Readability.js] by [Mozilla] and written line by line to make sure it looks and works as similar as possible. This way, hopefully all web page that can be parsed by Readability.js are parse-able by go-readability as well.
+
+> [!WARNING]
+> This package is deprecated in favor of [codeberg.org/readeck/go-readability/v2](https://codeberg.org/readeck/go-readability/src/branch/v2).
+
+## Table of Contents
+
+- [Table of Contents](#table-of-contents)
+- [Status](#status)
+- [Installation](#installation)
+- [Example](#example)
+- [Command Line Usage](#command-line-usage)
+- [Licenses](#licenses)
+
+## Status
+
+This package is stable enough for use and up to date with [Readability.js v0.5.0][last-version].
+
+For compatibility with Readability.js v0.6, use [codeberg.org/readeck/go-readability/v2](https://codeberg.org/readeck/go-readability/src/branch/v2).
+
+## Installation
+
+To install this package, just run `go get` :
+
+```
+go get -u -v github.com/go-shiori/go-readability
+```
+
+## Example
+
+To get the readable content from an URL, you can use `readability.FromURL`. It will fetch the web page from specified url, check if it's readable, then parses the response to find the readable content :
+
+```go
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	readability "github.com/go-shiori/go-readability"
+)
+
+var (
+	urls = []string{
+		// this one is article, so it's parse-able
+		"https://www.nytimes.com/2019/02/20/climate/climate-national-security-threat.html",
+		// while this one is not an article, so readability will fail to parse.
+		"https://www.nytimes.com/",
+	}
+)
+
+func main() {
+	for i, url := range urls {
+		article, err := readability.FromURL(url, 30*time.Second)
+		if err != nil {
+			log.Fatalf("failed to parse %s, %v\n", url, err)
+		}
+
+		dstTxtFile, _ := os.Create(fmt.Sprintf("text-%02d.txt", i+1))
+		defer dstTxtFile.Close()
+		dstTxtFile.WriteString(article.TextContent)
+
+		dstHTMLFile, _ := os.Create(fmt.Sprintf("html-%02d.html", i+1))
+		defer dstHTMLFile.Close()
+		dstHTMLFile.WriteString(article.Content)
+
+		fmt.Printf("URL     : %s\n", url)
+		fmt.Printf("Title   : %s\n", article.Title)
+		fmt.Printf("Author  : %s\n", article.Byline)
+		fmt.Printf("Length  : %d\n", article.Length)
+		fmt.Printf("Excerpt : %s\n", article.Excerpt)
+		fmt.Printf("SiteName: %s\n", article.SiteName)
+		fmt.Printf("Image   : %s\n", article.Image)
+		fmt.Printf("Favicon : %s\n", article.Favicon)
+		fmt.Printf("Text content saved to \"text-%02d.txt\"\n", i+1)
+		fmt.Printf("HTML content saved to \"html-%02d.html\"\n", i+1)
+		fmt.Println()
+	}
+}
+```
+
+However, sometimes you want to parse an URL no matter if it's an article or not. For example is when you only want to get metadata of the page. To do that, you have to download the page manually using `http.Get`, then parse it using `readability.FromReader` :
+
+```go
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+
+	readability "github.com/go-shiori/go-readability"
+)
+
+var (
+	urls = []string{
+		// Both will be parse-able now
+		"https://www.nytimes.com/2019/02/20/climate/climate-national-security-threat.html",
+		// But this one will not have any content
+		"https://www.nytimes.com/",
+	}
+)
+
+func main() {
+	for _, u := range urls {
+		resp, err := http.Get(u)
+		if err != nil {
+			log.Fatalf("failed to download %s: %v\n", u, err)
+		}
+		defer resp.Body.Close()
+
+		parsedURL, err := url.Parse(u)
+		if err != nil {
+			log.Fatalf("error parsing url")
+		}
+
+		article, err := readability.FromReader(resp.Body, parsedURL)
+		if err != nil {
+			log.Fatalf("failed to parse %s: %v\n", u, err)
+		}
+
+		fmt.Printf("URL     : %s\n", u)
+		fmt.Printf("Title   : %s\n", article.Title)
+		fmt.Printf("Author  : %s\n", article.Byline)
+		fmt.Printf("Length  : %d\n", article.Length)
+		fmt.Printf("Excerpt : %s\n", article.Excerpt)
+		fmt.Printf("SiteName: %s\n", article.SiteName)
+		fmt.Printf("Image   : %s\n", article.Image)
+		fmt.Printf("Favicon : %s\n", article.Favicon)
+		fmt.Println()
+	}
+}
+
+```
+
+## Command Line Usage
+
+You can also use `go-readability` as command line app. To do that, first install the CLI :
+
+```
+go install github.com/go-shiori/go-readability/cmd/go-readability@latest
+```
+
+Now you can use it by running `go-readability` in your terminal :
+
+```
+$ go-readability -h
+
+go-readability is parser to fetch the readable content of a web page.
+The source can be an url or existing file in your storage.
+
+Usage:
+  go-readability [flags] source
+
+Flags:
+  -h, --help          help for go-readability
+  -l, --http string   start the http server at the specified address
+  -m, --metadata      only print the page's metadata
+  -t, --text          only print the page's text
+```
+
+## Licenses
+
+Go-Readability is distributed under [MIT license][mit], which means you can use and modify it however you want. However, if you make an enhancement for it, if possible, please send a pull request. If you like this project, please consider donating to me either via [PayPal][paypal] or [Ko-Fi][kofi].
+
+[go-ref]: https://pkg.go.dev/github.com/go-shiori/go-readability
+[go-ref-badge]: https://img.shields.io/static/v1?label=&message=Reference&color=007d9c&logo=go&logoColor=white
+[paypal]: https://www.paypal.me/RadhiFadlillah
+[paypal-badge]: https://img.shields.io/static/v1?label=&message=PayPal&color=00457C&logo=paypal&logoColor=white
+[kofi]: https://ko-fi.com/radhifadlillah
+[kofi-badge]: https://img.shields.io/static/v1?label=&message=Ko-fi&color=F16061&logo=ko-fi&logoColor=white
+[readability.js]: https://github.com/mozilla/readability
+[mozilla]: https://github.com/mozilla
+[last-version]: https://github.com/mozilla/readability/tree/0.5.0
+[mit]: https://choosealicense.com/licenses/mit/

vendor/github.com/go-shiori/go-readability/internal/re2go/base.go

@@ -0,0 +1,2 @@
+// Code generated by re2go 4.0.2, DO NOT EDIT.
+package re2go

vendor/github.com/go-shiori/go-readability/internal/re2go/base.re

@@ -0,0 +1,18 @@
+package re2go
+
+/*!rules:re2c:base_template
+re2c:eof              = 0;
+re2c:yyfill:enable    = 0;
+re2c:posix-captures   = 0;
+re2c:case-insensitive = 0;
+
+re2c:define:YYCTYPE     = byte;
+re2c:define:YYPEEK      = "input[cursor]";
+re2c:define:YYSKIP      = "cursor++";
+re2c:define:YYBACKUP    = "marker = cursor";
+re2c:define:YYRESTORE   = "cursor = marker";
+re2c:define:YYLESSTHAN  = "limit <= cursor";
+re2c:define:YYSTAGP     = "@@{tag} = cursor";
+re2c:define:YYSTAGN     = "@@{tag} = -1";
+re2c:define:YYSHIFTSTAG = "@@{tag} += @@{shift}";
+*/

vendor/github.com/go-shiori/go-readability/internal/re2go/check-byline.go

@@ -0,0 +1,330 @@
+// Code generated by re2go 4.0.2, DO NOT EDIT.
+package re2go
+
+// Original pattern: (?i)byline|author|dateline|writtenby|p-author
+func IsByline(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for {
+		{
+			var yych byte
+			yych = input[cursor]
+			switch yych {
+			case 'A':
+				fallthrough
+			case 'a':
+				goto yy3
+			case 'B':
+				fallthrough
+			case 'b':
+				goto yy4
+			case 'D':
+				fallthrough
+			case 'd':
+				goto yy5
+			case 'P':
+				fallthrough
+			case 'p':
+				goto yy6
+			case 'W':
+				fallthrough
+			case 'w':
+				goto yy7
+			default:
+				if limit <= cursor {
+					goto yy29
+				}
+				goto yy1
+			}
+		yy1:
+			cursor++
+		yy2:
+			{
+				continue
+			}
+		yy3:
+			cursor++
+			marker = cursor
+			yych = input[cursor]
+			switch yych {
+			case 'U':
+				fallthrough
+			case 'u':
+				goto yy8
+			default:
+				goto yy2
+			}
+		yy4:
+			cursor++
+			marker = cursor
+			yych = input[cursor]
+			switch yych {
+			case 'Y':
+				fallthrough
+			case 'y':
+				goto yy10
+			default:
+				goto yy2
+			}
+		yy5:
+			cursor++
+			marker = cursor
+			yych = input[cursor]
+			switch yych {
+			case 'A':
+				fallthrough
+			case 'a':
+				goto yy11
+			default:
+				goto yy2
+			}
+		yy6:
+			cursor++
+			marker = cursor
+			yych = input[cursor]
+			switch yych {
+			case '-':
+				goto yy12
+			default:
+				goto yy2
+			}
+		yy7:
+			cursor++
+			marker = cursor
+			yych = input[cursor]
+			switch yych {
+			case 'R':
+				fallthrough
+			case 'r':
+				goto yy13
+			default:
+				goto yy2
+			}
+		yy8:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'T':
+				fallthrough
+			case 't':
+				goto yy14
+			default:
+				goto yy9
+			}
+		yy9:
+			cursor = marker
+			goto yy2
+		yy10:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'L':
+				fallthrough
+			case 'l':
+				goto yy15
+			default:
+				goto yy9
+			}
+		yy11:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'T':
+				fallthrough
+			case 't':
+				goto yy16
+			default:
+				goto yy9
+			}
+		yy12:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'A':
+				fallthrough
+			case 'a':
+				goto yy17
+			default:
+				goto yy9
+			}
+		yy13:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'I':
+				fallthrough
+			case 'i':
+				goto yy18
+			default:
+				goto yy9
+			}
+		yy14:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'H':
+				fallthrough
+			case 'h':
+				goto yy19
+			default:
+				goto yy9
+			}
+		yy15:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'I':
+				fallthrough
+			case 'i':
+				goto yy20
+			default:
+				goto yy9
+			}
+		yy16:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'E':
+				fallthrough
+			case 'e':
+				goto yy10
+			default:
+				goto yy9
+			}
+		yy17:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'U':
+				fallthrough
+			case 'u':
+				goto yy8
+			default:
+				goto yy9
+			}
+		yy18:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'T':
+				fallthrough
+			case 't':
+				goto yy21
+			default:
+				goto yy9
+			}
+		yy19:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'O':
+				fallthrough
+			case 'o':
+				goto yy22
+			default:
+				goto yy9
+			}
+		yy20:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'N':
+				fallthrough
+			case 'n':
+				goto yy23
+			default:
+				goto yy9
+			}
+		yy21:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'T':
+				fallthrough
+			case 't':
+				goto yy24
+			default:
+				goto yy9
+			}
+		yy22:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'R':
+				fallthrough
+			case 'r':
+				goto yy25
+			default:
+				goto yy9
+			}
+		yy23:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'E':
+				fallthrough
+			case 'e':
+				goto yy25
+			default:
+				goto yy9
+			}
+		yy24:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'E':
+				fallthrough
+			case 'e':
+				goto yy26
+			default:
+				goto yy9
+			}
+		yy25:
+			cursor++
+			{
+				return true
+			}
+		yy26:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'N':
+				fallthrough
+			case 'n':
+				goto yy27
+			default:
+				goto yy9
+			}
+		yy27:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'B':
+				fallthrough
+			case 'b':
+				goto yy28
+			default:
+				goto yy9
+			}
+		yy28:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case 'Y':
+				fallthrough
+			case 'y':
+				goto yy25
+			default:
+				goto yy9
+			}
+		yy29:
+			{
+				return false
+			}
+		}
+
+	}
+}

vendor/github.com/go-shiori/go-readability/internal/re2go/check-byline.re

@@ -0,0 +1,20 @@
+/*!include:re2c "base.re" */
+
+// Original pattern: (?i)byline|author|dateline|writtenby|p-author
+func IsByline(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for { /*!use:re2c:base_template
+		re2c:case-insensitive = 1;
+
+		byline = byline|author|dateline|writtenby|p-author;
+
+		{byline} { return true }
+		*        { continue }
+		$        { return false }
+		*/
+	}
+}

vendor/github.com/go-shiori/go-readability/internal/re2go/class-weight.re

@@ -0,0 +1,100 @@
+/*!include:re2c "base.re" */
+
+// Original pattern: (?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story
+func IsPositiveClass(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for { /*!use:re2c:base_template
+		re2c:case-insensitive = 1;
+
+		positive = article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story;
+
+		{positive} { return true }
+		*          { continue }
+		$          { return false }
+		*/
+	}
+}
+
+// For IsNegativeClass, its original pattern is like this:
+// (?i)-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget
+//
+// Unfortunately, re2go doesn't handle anchor like ^ and $ internally, so for convenience
+// I'll split that pattern into two:
+// - `^hid$| hid$| hid |^hid `
+// - `-ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget`
+func IsNegativeClass(input string) bool {
+	return isNegativeClass1(input) || isNegativeClass2(input)
+}
+
+// This one handle: `^hid$| hid$| hid |^hid `
+func isNegativeClass1(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	// Variable for capturing parentheses (twice the number of groups).
+	/*!maxnmatch:re2c*/
+	yypmatch := make([]int, YYMAXNMATCH*2)
+	var yynmatch int
+	_ = yynmatch
+
+	// Autogenerated tag variables used by the lexer to track tag values.
+	/*!stags:re2c format = 'var @@ int; _ = @@\n'; */
+
+	for { /*!use:re2c:base_template
+		re2c:posix-captures   = 1;
+		re2c:case-insensitive = 1;
+
+		(.)?hid(.)? {
+			// Extract submatch first
+			var sub1, sub2 string
+			start, end := yypmatch[0], yypmatch[1]
+
+			if yypmatch[2] != -1 {
+				sub1 = input[yypmatch[2]:yypmatch[3]]
+			}
+
+			if yypmatch[4] != -1 {
+				sub2 = input[yypmatch[4]:yypmatch[5]]
+			}
+
+			case1 := start == 0 && end == limit && sub1 == "" && sub2 == "" // '^hid$'
+			case2 := end == limit && sub1 == " " && sub2 == "" // ' hid$'
+			case3 := start == 0 && sub1 == "" && sub2 == " " // '^hid '
+			case4 := sub1 == " " && sub2 == " " // ' hid '
+
+			if case1 || case2 || case3 || case4 {
+				return true
+			}
+			continue
+		}
+
+		* { continue }
+		$ { return false }
+		*/
+	}
+}
+
+// This one handle: `-ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget`
+func isNegativeClass2(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for { /*!use:re2c:base_template
+		re2c:case-insensitive = 1;
+
+		negative = -ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget;
+
+		{negative} { return true }
+		*          { continue }
+		$          { return false }
+		*/
+	}
+}

vendor/github.com/go-shiori/go-readability/internal/re2go/grab-article.re

@@ -0,0 +1,61 @@
+/*!include:re2c "base.re" */
+
+// Original pattern: (?i)-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote
+func IsUnlikelyCandidates(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for { /*!use:re2c:base_template
+		re2c:case-insensitive = 1;
+
+		unlikely = -ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote;
+
+		{unlikely} { return true }
+		*          { continue }
+		$          { return false }
+		*/
+	}
+}
+
+// Original pattern: (?i)and|article|body|column|content|main|shadow
+func MaybeItsACandidate(input string) bool {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for { /*!use:re2c:base_template
+		re2c:case-insensitive = 1;
+
+		maybe = and|article|body|column|content|main|shadow;
+
+		{maybe} { return true }
+		*       { continue }
+		$       { return false }
+		*/
+	}
+}
+
+// Commas as used in Latin, Sindhi, Chinese and various other scripts.
+// see: https://en.wikipedia.org/wiki/Comma#Comma_variants
+// Original pattern: \u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C
+func CountCommas(input string) int {
+	var count int
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	for { /*!use:re2c:base_template
+		re2c:case-insensitive = 1;
+
+		commas = [\u002C\u060C\uFE50\uFE10\uFE11\u2E41\u2E34\u2E32\uFF0C];
+
+		{commas} { count++; continue }
+		*        { continue }
+		$        { return count }
+		*/
+	}
+}

vendor/github.com/go-shiori/go-readability/internal/re2go/normalize.go

@@ -0,0 +1,95 @@
+// Code generated by re2go 4.0.2, DO NOT EDIT.
+package re2go
+
+import "strings"
+
+// Original pattern: (?i)\s{2,}
+func NormalizeSpaces(input string) string {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	// Variable for capturing parentheses (twice the number of groups).
+	const YYMAXNMATCH = 1
+
+	yypmatch := make([]int, YYMAXNMATCH*2)
+	var yynmatch int
+	_ = yynmatch
+
+	// Autogenerated tag variables used by the lexer to track tag values.
+	var yyt1 int
+	_ = yyt1
+
+	var start int
+	var sb strings.Builder
+
+	for {
+		{
+			var yych byte
+			yych = input[cursor]
+			switch yych {
+			case '\t', '\n':
+				fallthrough
+			case '\f', '\r':
+				fallthrough
+			case ' ':
+				yyt1 = cursor
+				goto yy3
+			default:
+				if limit <= cursor {
+					goto yy6
+				}
+				goto yy1
+			}
+		yy1:
+			cursor++
+		yy2:
+			{
+				continue
+			}
+		yy3:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case '\t', '\n':
+				fallthrough
+			case '\f', '\r':
+				fallthrough
+			case ' ':
+				goto yy4
+			default:
+				goto yy2
+			}
+		yy4:
+			cursor++
+			yych = input[cursor]
+			switch yych {
+			case '\t', '\n':
+				fallthrough
+			case '\f', '\r':
+				fallthrough
+			case ' ':
+				goto yy4
+			default:
+				goto yy5
+			}
+		yy5:
+			yynmatch = 1
+			yypmatch[0] = yyt1
+			yypmatch[1] = cursor
+			{
+				sb.WriteString(input[start:yypmatch[0]])
+				sb.WriteString(" ")
+				start = yypmatch[1]
+				continue
+			}
+		yy6:
+			{
+				sb.WriteString(input[start:limit])
+				return sb.String()
+			}
+		}
+
+	}
+}

vendor/github.com/go-shiori/go-readability/internal/re2go/normalize.re

@@ -0,0 +1,42 @@
+/*!include:re2c "base.re" */
+
+import "strings"
+
+// Original pattern: (?i)\s{2,}
+func NormalizeSpaces(input string) string {
+	var cursor, marker int
+	input += string(rune(0)) // add terminating null
+	limit := len(input) - 1  // limit points at the terminating null
+	_ = marker
+
+	// Variable for capturing parentheses (twice the number of groups).
+	/*!maxnmatch:re2c*/
+	yypmatch := make([]int, YYMAXNMATCH*2)
+	var yynmatch int
+	_ = yynmatch
+
+	// Autogenerated tag variables used by the lexer to track tag values.
+	/*!stags:re2c format = 'var @@ int; _ = @@\n'; */
+
+	var start int
+	var sb strings.Builder
+
+	for { /*!use:re2c:base_template
+		re2c:posix-captures = 1;
+
+		[\t\n\f\r ]{2,} {
+			sb.WriteString(input[start:yypmatch[0]])
+			sb.WriteString(" ")
+			start = yypmatch[1]
+			continue
+		}
+
+		$ {
+			sb.WriteString(input[start:limit])
+			return sb.String()
+		}
+
+		* { continue }
+		*/
+	}
+}

vendor/github.com/go-shiori/go-readability/parser-check.go

@@ -0,0 +1,80 @@
+package readability
+
+import (
+	"io"
+	"math"
+	"strings"
+
+	"github.com/go-shiori/dom"
+	"github.com/go-shiori/go-readability/internal/re2go"
+	"golang.org/x/net/html"
+)
+
+// Check checks whether the input is readable without parsing the whole thing.
+func (ps *Parser) Check(input io.Reader) bool {
+	// Parse input
+	doc, err := dom.Parse(input)
+	if err != nil {
+		return false
+	}
+
+	return ps.CheckDocument(doc)
+}
+
+// CheckDocument checks whether the document is readable without parsing the whole thing.
+func (ps *Parser) CheckDocument(doc *html.Node) bool {
+	// Get <p> and <pre> nodes.
+	nodes := dom.QuerySelectorAll(doc, "p, pre, article")
+
+	// Also get <div> nodes which have <br> node(s) and append
+	// them into the `nodes` variable.
+	// Some articles' DOM structures might look like :
+	//
+	// <div>
+	//     Sentences<br>
+	//     <br>
+	//     Sentences<br>
+	// </div>
+	//
+	// So we need to make sure only fetch the div once.
+	// To do so, we will use map as dictionary.
+	tracker := make(map[*html.Node]struct{})
+	for _, br := range dom.QuerySelectorAll(doc, "div > br") {
+		if br.Parent == nil {
+			continue
+		}
+
+		if _, exist := tracker[br.Parent]; !exist {
+			tracker[br.Parent] = struct{}{}
+			nodes = append(nodes, br.Parent)
+		}
+	}
+
+	// This is a little cheeky, we use the accumulator 'score' to decide what
+	// to return from this callback.
+	score := float64(0)
+	return ps.someNode(nodes, func(node *html.Node) bool {
+		if !ps.isProbablyVisible(node) {
+			return false
+		}
+
+		matchString := dom.ClassName(node) + " " + dom.ID(node)
+		if re2go.IsUnlikelyCandidates(matchString) &&
+			!re2go.MaybeItsACandidate(matchString) {
+			return false
+		}
+
+		if dom.TagName(node) == "p" && ps.hasAncestorTag(node, "li", -1, nil) {
+			return false
+		}
+
+		nodeText := strings.TrimSpace(dom.TextContent(node))
+		nodeTextLength := len(nodeText)
+		if nodeTextLength < 140 {
+			return false
+		}
+
+		score += math.Sqrt(float64(nodeTextLength - 140))
+		return score > 20
+	})
+}

vendor/github.com/go-shiori/go-readability/parser-parse.go

@@ -0,0 +1,156 @@
+package readability
+
+import (
+	"fmt"
+	"io"
+	nurl "net/url"
+	"strings"
+	"time"
+
+	"github.com/araddon/dateparse"
+	"github.com/go-shiori/dom"
+	"golang.org/x/net/html"
+)
+
+// Parse parses a reader and find the main readable content.
+func (ps *Parser) Parse(input io.Reader, pageURL *nurl.URL) (Article, error) {
+	// Parse input
+	doc, err := dom.Parse(input)
+	if err != nil {
+		return Article{}, fmt.Errorf("failed to parse input: %v", err)
+	}
+
+	return ps.ParseDocument(doc, pageURL)
+}
+
+// ParseDocument parses the specified document and find the main readable content.
+func (ps *Parser) ParseDocument(doc *html.Node, pageURL *nurl.URL) (Article, error) {
+	// Clone document to make sure the original kept untouched
+	ps.doc = dom.Clone(doc, true)
+
+	// Reset parser data
+	ps.articleTitle = ""
+	ps.articleByline = ""
+	ps.articleDir = ""
+	ps.articleSiteName = ""
+	ps.documentURI = pageURL
+	ps.attempts = []parseAttempt{}
+	ps.flags = flags{
+		stripUnlikelys:     true,
+		useWeightClasses:   true,
+		cleanConditionally: true,
+	}
+
+	// Avoid parsing too large documents, as per configuration option
+	if ps.MaxElemsToParse > 0 {
+		numTags := len(dom.GetElementsByTagName(ps.doc, "*"))
+		if numTags > ps.MaxElemsToParse {
+			return Article{}, fmt.Errorf("documents too large: %d elements", numTags)
+		}
+	}
+
+	// Unwrap image from noscript
+	ps.unwrapNoscriptImages(ps.doc)
+
+	// Extract JSON-LD metadata before removing scripts
+	var jsonLd map[string]string
+	if !ps.DisableJSONLD {
+		jsonLd, _ = ps.getJSONLD()
+	}
+
+	// Remove script tags from the document.
+	ps.removeScripts(ps.doc)
+
+	// Prepares the HTML document
+	ps.prepDocument()
+
+	// Fetch metadata
+	metadata := ps.getArticleMetadata(jsonLd)
+	ps.articleTitle = metadata["title"]
+
+	// Try to grab article content
+	finalHTMLContent := ""
+	finalTextContent := ""
+	articleContent := ps.grabArticle()
+	var readableNode *html.Node
+
+	if articleContent != nil {
+		ps.postProcessContent(articleContent)
+
+		// If we haven't found an excerpt in the article's metadata,
+		// use the article's first paragraph as the excerpt. This is used
+		// for displaying a preview of the article's content.
+		if metadata["excerpt"] == "" {
+			paragraphs := dom.GetElementsByTagName(articleContent, "p")
+			if len(paragraphs) > 0 {
+				metadata["excerpt"] = strings.TrimSpace(dom.TextContent(paragraphs[0]))
+			}
+		}
+
+		readableNode = dom.FirstElementChild(articleContent)
+		finalHTMLContent = dom.InnerHTML(articleContent)
+		finalTextContent = dom.TextContent(articleContent)
+		finalTextContent = strings.TrimSpace(finalTextContent)
+	}
+
+	finalByline := metadata["byline"]
+	if finalByline == "" {
+		finalByline = ps.articleByline
+	}
+
+	// Excerpt is an supposed to be short and concise,
+	// so it shouldn't have any new line
+	excerpt := strings.TrimSpace(metadata["excerpt"])
+	excerpt = strings.Join(strings.Fields(excerpt), " ")
+
+	// go-readability special:
+	// Internet is dangerous and weird, and sometimes we will find
+	// metadata isn't encoded using a valid Utf-8, so here we check it.
+	var replacementTitle string
+	if pageURL != nil {
+		replacementTitle = pageURL.String()
+	}
+
+	validTitle := strings.ToValidUTF8(ps.articleTitle, replacementTitle)
+	validByline := strings.ToValidUTF8(finalByline, "")
+	validExcerpt := strings.ToValidUTF8(excerpt, "")
+
+	publishedTime := ps.getDate(metadata, "publishedTime")
+	modifiedTime := ps.getDate(metadata, "modifiedTime")
+
+	return Article{
+		Title:         validTitle,
+		Byline:        validByline,
+		Node:          readableNode,
+		Content:       finalHTMLContent,
+		TextContent:   finalTextContent,
+		Length:        charCount(finalTextContent),
+		Excerpt:       validExcerpt,
+		SiteName:      metadata["siteName"],
+		Image:         metadata["image"],
+		Favicon:       metadata["favicon"],
+		Language:      ps.articleLang,
+		PublishedTime: publishedTime,
+		ModifiedTime:  modifiedTime,
+	}, nil
+}
+
+// getDate tries to get a date from metadata, and parse it using a list of known formats.
+func (ps *Parser) getDate(metadata map[string]string, fieldName string) *time.Time {
+	dateStr, ok := metadata[fieldName]
+	if ok && len(dateStr) > 0 {
+		return ps.getParsedDate(dateStr)
+	}
+	return nil
+}
+
+// getParsedDate tries to parse a date string using a list of known formats.
+// If the date string can't be parsed, it will return nil.
+func (ps *Parser) getParsedDate(dateStr string) *time.Time {
+	d, err := dateparse.ParseAny(dateStr)
+	if err != nil {
+		ps.logf("failed to parse date \"%s\": %v\n", dateStr, err)
+		return nil
+	}
+	return &d
+}

vendor/github.com/go-shiori/go-readability/readability.go

@@ -0,0 +1,86 @@
+// Package readability is a Go package that find the main readable
+// content from a HTML page. It works by removing clutter like buttons,
+// ads, background images, script, etc.
+//
+// This package is based from Readability.js by Mozilla, and written line
+// by line to make sure it looks and works as similar as possible. This
+// way, hopefully all web page that can be parsed by Readability.js
+// are parse-able by go-readability as well.
+package readability
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	nurl "net/url"
+	"strings"
+	"time"
+
+	"golang.org/x/net/html"
+)
+
+// FromReader parses an `io.Reader` and returns the readable content. It's the wrapper
+// or `Parser.Parse()` and useful if you only want to use the default parser.
+func FromReader(input io.Reader, pageURL *nurl.URL) (Article, error) {
+	parser := NewParser()
+	return parser.Parse(input, pageURL)
+}
+
+// FromDocument parses an document and returns the readable content. It's the wrapper
+// or `Parser.ParseDocument()` and useful if you only want to use the default parser.
+func FromDocument(doc *html.Node, pageURL *nurl.URL) (Article, error) {
+	parser := NewParser()
+	return parser.ParseDocument(doc, pageURL)
+}
+
+type RequestWith func(r *http.Request)
+
+// FromURL fetch the web page from specified url then parses the response to find
+// the readable content.
+func FromURL(pageURL string, timeout time.Duration, requestModifiers ...RequestWith) (Article, error) {
+	// Make sure URL is valid
+	parsedURL, err := nurl.ParseRequestURI(pageURL)
+	if err != nil {
+		return Article{}, fmt.Errorf("failed to parse URL: %v", err)
+	}
+
+	// Fetch page from URL
+	client := &http.Client{Timeout: timeout}
+	req, err := http.NewRequest("GET", pageURL, nil)
+	for _, modifer := range requestModifiers {
+		modifer(req)
+	}
+	if err != nil {
+		return Article{}, fmt.Errorf("failed to fetch the page: %v", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return Article{}, fmt.Errorf("failed to fetch the page: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Make sure content type is HTML
+	cp := resp.Header.Get("Content-Type")
+	if !strings.Contains(cp, "text/html") {
+		return Article{}, fmt.Errorf("URL is not a HTML document")
+	}
+
+	// Parse content
+	parser := NewParser()
+	return parser.Parse(resp.Body, parsedURL)
+}
+
+// Check checks whether the input is readable without parsing the whole thing. It's the
+// wrapper for `Parser.Check()` and useful if you only use the default parser.
+func Check(input io.Reader) bool {
+	parser := NewParser()
+	return parser.Check(input)
+}
+
+// CheckDocument checks whether the document is readable without parsing the whole thing.
+// It's the wrapper for `Parser.CheckDocument()` and useful if you only use the default
+// parser.
+func CheckDocument(doc *html.Node) bool {
+	parser := NewParser()
+	return parser.CheckDocument(doc)
+}

vendor/github.com/go-shiori/go-readability/utils.go

@@ -0,0 +1,100 @@
+package readability
+
+import (
+	nurl "net/url"
+	"strings"
+	"unicode/utf8"
+)
+
+// indexOf returns the position of the first occurrence of a
+// specified  value in a string array. Returns -1 if the
+// value to search for never occurs.
+func indexOf(array []string, key string) int {
+	for i := 0; i < len(array); i++ {
+		if array[i] == key {
+			return i
+		}
+	}
+	return -1
+}
+
+// wordCount returns number of word in str.
+func wordCount(str string) int {
+	return len(strings.Fields(str))
+}
+
+// charCount returns number of char in str.
+func charCount(str string) int {
+	return utf8.RuneCountInString(str)
+}
+
+// isValidURL checks if URL is valid.
+func isValidURL(s string) bool {
+	_, err := nurl.ParseRequestURI(s)
+	return err == nil
+}
+
+// toAbsoluteURI convert uri to absolute path based on base.
+// However, if uri is prefixed with hash (#), the uri won't be changed.
+func toAbsoluteURI(uri string, base *nurl.URL) string {
+	if uri == "" || base == nil {
+		return uri
+	}
+
+	// If it is hash tag, return as it is
+	if strings.HasPrefix(uri, "#") {
+		return uri
+	}
+
+	// If it is data URI, return as it is
+	if strings.HasPrefix(uri, "data:") {
+		return uri
+	}
+
+	// If it is already an absolute URL, return as it is
+	tmp, err := nurl.ParseRequestURI(uri)
+	if err == nil && tmp.Scheme != "" && tmp.Hostname() != "" {
+		return uri
+	}
+
+	// Otherwise, resolve against base URI.
+	tmp, err = nurl.Parse(uri)
+	if err != nil {
+		return uri
+	}
+
+	return base.ResolveReference(tmp).String()
+}
+
+// strOr returns the first not empty string in args.
+func strOr(args ...string) string {
+	for i := 0; i < len(args); i++ {
+		if args[i] != "" {
+			return args[i]
+		}
+	}
+	return ""
+}
+
+func sliceToMap(strings ...string) map[string]struct{} {
+	result := make(map[string]struct{})
+	for _, s := range strings {
+		result[s] = struct{}{}
+	}
+	return result
+}
+
+func strFilter(strs []string, filter func(string) bool) []string {
+	var result []string
+	for _, s := range strs {
+		if filter(s) {
+			result = append(result, s)
+		}
+	}
+	return result
+}
+
+func trim(s string) string {
+	s = strings.Join(strings.Fields(s), " ")
+	return strings.TrimSpace(s)
+}