p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests
Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
This commit is contained in:
+174
@@ -0,0 +1,174 @@
|
||||
/*
|
||||
|
||||
The function to collapse whitespace was adapted from the "turndown" library by Dom Christie,
|
||||
which was adapted from the "collapse-whitespace" library by Luc Thevenard.
|
||||
|
||||
It was ported from Javascript to Golang by Johannes Kaufmann for the use in the "html-to-markdown" library.
|
||||
To increase performance the use of regex was replaced by custom code.
|
||||
|
||||
https://github.com/wooorm/collapse-white-space
|
||||
https://github.com/mixmark-io/turndown
|
||||
https://github.com/JohannesKaufmann/html-to-markdown
|
||||
|
||||
-----------
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2017 Dom Christie
|
||||
Copyright (c) 2014 Luc Thevenard <lucthevenard@gmail.com>
|
||||
Copyright (c) 2018 Johannes Kaufmann
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
*/
|
||||
|
||||
// collapse can collapse whitespace in html elements.
|
||||
//
|
||||
// It is a port from the Javascript library "turndown" to Golang.
|
||||
package collapse
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func nextNode(prev *html.Node, current *html.Node, domFuncs *DomFuncs) *html.Node {
|
||||
if (prev != nil && prev.Parent == current) || domFuncs.IsPreformattedNode(current) {
|
||||
if current.NextSibling != nil {
|
||||
return current.NextSibling
|
||||
}
|
||||
|
||||
return current.Parent
|
||||
}
|
||||
|
||||
if current.FirstChild != nil {
|
||||
return current.FirstChild
|
||||
}
|
||||
if current.NextSibling != nil {
|
||||
return current.NextSibling
|
||||
}
|
||||
|
||||
return current.Parent
|
||||
}
|
||||
|
||||
func removeNode(node *html.Node) *html.Node {
|
||||
next := node.NextSibling
|
||||
if next == nil {
|
||||
next = node.Parent
|
||||
}
|
||||
|
||||
node.Parent.RemoveChild(node)
|
||||
|
||||
return next
|
||||
|
||||
}
|
||||
|
||||
type DomFuncs struct {
|
||||
IsBlockNode func(node *html.Node) bool
|
||||
IsVoidNode func(node *html.Node) bool
|
||||
IsPreformattedNode func(node *html.Node) bool
|
||||
}
|
||||
|
||||
func fillDefaultDomFuncs(domFuncs *DomFuncs) *DomFuncs {
|
||||
if domFuncs == nil {
|
||||
domFuncs = &DomFuncs{}
|
||||
}
|
||||
if domFuncs.IsBlockNode == nil {
|
||||
domFuncs.IsBlockNode = defaultIsBlockNode
|
||||
}
|
||||
if domFuncs.IsVoidNode == nil {
|
||||
domFuncs.IsVoidNode = defaultIsVoidNode
|
||||
}
|
||||
if domFuncs.IsPreformattedNode == nil {
|
||||
domFuncs.IsPreformattedNode = defaultIsPreformattedNode
|
||||
}
|
||||
return domFuncs
|
||||
|
||||
}
|
||||
func Collapse(element *html.Node, domFuncs *DomFuncs) {
|
||||
domFuncs = fillDefaultDomFuncs(domFuncs)
|
||||
// - - - - - - - - - - - - - - - - - - //
|
||||
|
||||
if element.FirstChild == nil || domFuncs.IsPreformattedNode(element) {
|
||||
return
|
||||
}
|
||||
|
||||
var prevText *html.Node = nil
|
||||
var keepLeadingWs = false
|
||||
|
||||
var prev *html.Node = nil
|
||||
var node = nextNode(prev, element, domFuncs)
|
||||
|
||||
for node != element {
|
||||
if node.Type == html.TextNode /* node.nodeType == 4 */ { // Node.TEXT_NODE or Node.CDATA_SECTION_NODE
|
||||
var text = replaceAnyWhitespaceWithSpace(node.Data)
|
||||
|
||||
if (prevText == nil || strings.HasSuffix(prevText.Data, " ")) &&
|
||||
!keepLeadingWs && text != "" && text[0] == ' ' {
|
||||
text = text[1:]
|
||||
}
|
||||
|
||||
// `text` might be empty at this point.
|
||||
if text == "" {
|
||||
node = removeNode(node)
|
||||
continue
|
||||
}
|
||||
|
||||
node.Data = text
|
||||
|
||||
prevText = node
|
||||
} else if node.Type == html.ElementNode { // Node.ELEMENT_NODE
|
||||
if domFuncs.IsBlockNode(node) || dom.NodeName(node) == "br" {
|
||||
if prevText != nil {
|
||||
prevText.Data = strings.TrimSuffix(prevText.Data, " ")
|
||||
}
|
||||
|
||||
prevText = nil
|
||||
keepLeadingWs = false
|
||||
} else if domFuncs.IsVoidNode(node) || domFuncs.IsPreformattedNode(node) || dom.NodeName(node) == "code" {
|
||||
// Avoid trimming space around non-block, non-BR void elements and inline PRE.
|
||||
prevText = nil
|
||||
keepLeadingWs = true
|
||||
} else if prevText != nil {
|
||||
// Drop protection if set previously.
|
||||
keepLeadingWs = false
|
||||
}
|
||||
} else if node.Type == html.CommentNode {
|
||||
// TODO: Is this enough to keep the comments? Does this cause other problems?
|
||||
} else {
|
||||
// E.g. DoctypeNode
|
||||
|
||||
node = removeNode(node)
|
||||
continue
|
||||
}
|
||||
|
||||
var nextNode = nextNode(prev, node, domFuncs)
|
||||
prev = node
|
||||
node = nextNode
|
||||
}
|
||||
|
||||
if prevText != nil {
|
||||
prevText.Data = strings.TrimSuffix(prevText.Data, " ")
|
||||
if prevText.Data == "" {
|
||||
removeNode(prevText)
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package collapse
|
||||
|
||||
import (
|
||||
"github.com/JohannesKaufmann/dom"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
var blockElements = []string{
|
||||
"address", "article", "aside", "audio", "blockquote", "body", "canvas", "center", "dd", "dir", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "frameset", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "html", "isindex", "li", "main", "menu", "nav", "noframes", "noscript", "ol", "output", "p", "pre", "section", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul",
|
||||
}
|
||||
|
||||
var voidElements = []string{
|
||||
// Note: Compared to the javascript implementation, I removed "source"
|
||||
"area", "base", "br", "col", "command", "embed", "hr", "img", "input", "keygen", "link", "meta", "param" /* "source, "*/, "track", "wbr",
|
||||
}
|
||||
|
||||
var defaultIsBlockNode = func(node *html.Node) bool {
|
||||
name := dom.NodeName(node)
|
||||
|
||||
for _, elem := range blockElements {
|
||||
if elem == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
var defaultIsVoidNode = func(node *html.Node) bool {
|
||||
name := dom.NodeName(node)
|
||||
|
||||
for _, elem := range voidElements {
|
||||
if elem == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
var defaultIsPreformattedNode = func(node *html.Node) bool {
|
||||
// Note: Originally in the javascript version, this just checked for "pre".
|
||||
// I changed it, to also return true for "code"
|
||||
name := dom.NodeName(node)
|
||||
|
||||
return name == "pre" || name == "code"
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package collapse
|
||||
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func byteSliceToString(b []byte) string {
|
||||
/* #nosec G103 */
|
||||
return *(*string)(unsafe.Pointer(&b))
|
||||
}
|
||||
|
||||
func replaceAnyWhitespaceWithSpace(source string) string {
|
||||
if source == "" {
|
||||
return source
|
||||
}
|
||||
|
||||
// Some performance optimizations:
|
||||
// - If no replacement was done, we return the original slice and dont allocate.
|
||||
// - We batch appends
|
||||
var ret []byte
|
||||
makeIfNeeded := func() {
|
||||
if ret == nil {
|
||||
ret = make([]byte, 0, len(source))
|
||||
}
|
||||
}
|
||||
|
||||
startNormal := 0
|
||||
startMatch := -1
|
||||
for i := 0; i < len(source); i++ {
|
||||
isWhitespace := source[i] == ' ' || source[i] == '\r' || source[i] == '\n' || source[i] == '\t'
|
||||
|
||||
if startMatch == -1 && isWhitespace {
|
||||
// Start of newlines
|
||||
startMatch = i
|
||||
continue
|
||||
} else if startMatch != -1 && isWhitespace {
|
||||
// Middle of newlines
|
||||
continue
|
||||
} else if startMatch != -1 {
|
||||
// Character after the last newline character
|
||||
|
||||
count := i - startMatch
|
||||
if count == 1 && source[startMatch] == ' ' {
|
||||
// There was only one `isWhitespace` match & that is a space.
|
||||
// So the replacement would be exactly the same...
|
||||
} else {
|
||||
makeIfNeeded()
|
||||
ret = append(ret, source[startNormal:startMatch]...)
|
||||
ret = append(ret, byte(' '))
|
||||
startNormal = i
|
||||
}
|
||||
|
||||
startMatch = -1
|
||||
}
|
||||
}
|
||||
|
||||
if startMatch == -1 && startNormal == 0 {
|
||||
// a) no changes need to be done
|
||||
} else if startMatch == -1 {
|
||||
// b) Only the normal characters until the end still need to be added
|
||||
makeIfNeeded()
|
||||
ret = append(ret, source[startNormal:]...)
|
||||
} else if ret == nil && len(source)-startMatch == 1 && source[startMatch] == ' ' {
|
||||
// c) There is a match, but it is exactly the same as the replacement
|
||||
// If there is no new slice, we can skip the replacement.
|
||||
} else {
|
||||
// d) The match still needs to be replaced (and possible the previous normal characters be added)
|
||||
makeIfNeeded()
|
||||
ret = append(ret, source[startNormal:startMatch]...)
|
||||
ret = append(ret, byte(' '))
|
||||
}
|
||||
|
||||
if ret == nil {
|
||||
// Huray, we did not do any allocations with make()
|
||||
// and instead just return the original slice.
|
||||
return source
|
||||
}
|
||||
return byteSliceToString(ret)
|
||||
}
|
||||
Reference in New Issue
Block a user