p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests
Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
This commit is contained in:
+2
@@ -0,0 +1,2 @@
|
||||
// Code generated by re2go 4.0.2, DO NOT EDIT.
|
||||
package re2go
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package re2go
|
||||
|
||||
/*!rules:re2c:base_template
|
||||
re2c:eof = 0;
|
||||
re2c:yyfill:enable = 0;
|
||||
re2c:posix-captures = 0;
|
||||
re2c:case-insensitive = 0;
|
||||
|
||||
re2c:define:YYCTYPE = byte;
|
||||
re2c:define:YYPEEK = "input[cursor]";
|
||||
re2c:define:YYSKIP = "cursor++";
|
||||
re2c:define:YYBACKUP = "marker = cursor";
|
||||
re2c:define:YYRESTORE = "cursor = marker";
|
||||
re2c:define:YYLESSTHAN = "limit <= cursor";
|
||||
re2c:define:YYSTAGP = "@@{tag} = cursor";
|
||||
re2c:define:YYSTAGN = "@@{tag} = -1";
|
||||
re2c:define:YYSHIFTSTAG = "@@{tag} += @@{shift}";
|
||||
*/
|
||||
+330
@@ -0,0 +1,330 @@
|
||||
// Code generated by re2go 4.0.2, DO NOT EDIT.
|
||||
package re2go
|
||||
|
||||
// Original pattern: (?i)byline|author|dateline|writtenby|p-author
|
||||
func IsByline(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for {
|
||||
{
|
||||
var yych byte
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'A':
|
||||
fallthrough
|
||||
case 'a':
|
||||
goto yy3
|
||||
case 'B':
|
||||
fallthrough
|
||||
case 'b':
|
||||
goto yy4
|
||||
case 'D':
|
||||
fallthrough
|
||||
case 'd':
|
||||
goto yy5
|
||||
case 'P':
|
||||
fallthrough
|
||||
case 'p':
|
||||
goto yy6
|
||||
case 'W':
|
||||
fallthrough
|
||||
case 'w':
|
||||
goto yy7
|
||||
default:
|
||||
if limit <= cursor {
|
||||
goto yy29
|
||||
}
|
||||
goto yy1
|
||||
}
|
||||
yy1:
|
||||
cursor++
|
||||
yy2:
|
||||
{
|
||||
continue
|
||||
}
|
||||
yy3:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'U':
|
||||
fallthrough
|
||||
case 'u':
|
||||
goto yy8
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy4:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'Y':
|
||||
fallthrough
|
||||
case 'y':
|
||||
goto yy10
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy5:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'A':
|
||||
fallthrough
|
||||
case 'a':
|
||||
goto yy11
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy6:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case '-':
|
||||
goto yy12
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy7:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'R':
|
||||
fallthrough
|
||||
case 'r':
|
||||
goto yy13
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy8:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'T':
|
||||
fallthrough
|
||||
case 't':
|
||||
goto yy14
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy9:
|
||||
cursor = marker
|
||||
goto yy2
|
||||
yy10:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'L':
|
||||
fallthrough
|
||||
case 'l':
|
||||
goto yy15
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy11:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'T':
|
||||
fallthrough
|
||||
case 't':
|
||||
goto yy16
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy12:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'A':
|
||||
fallthrough
|
||||
case 'a':
|
||||
goto yy17
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy13:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'I':
|
||||
fallthrough
|
||||
case 'i':
|
||||
goto yy18
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy14:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'H':
|
||||
fallthrough
|
||||
case 'h':
|
||||
goto yy19
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy15:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'I':
|
||||
fallthrough
|
||||
case 'i':
|
||||
goto yy20
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy16:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'E':
|
||||
fallthrough
|
||||
case 'e':
|
||||
goto yy10
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy17:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'U':
|
||||
fallthrough
|
||||
case 'u':
|
||||
goto yy8
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy18:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'T':
|
||||
fallthrough
|
||||
case 't':
|
||||
goto yy21
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy19:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'O':
|
||||
fallthrough
|
||||
case 'o':
|
||||
goto yy22
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy20:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'N':
|
||||
fallthrough
|
||||
case 'n':
|
||||
goto yy23
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy21:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'T':
|
||||
fallthrough
|
||||
case 't':
|
||||
goto yy24
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy22:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'R':
|
||||
fallthrough
|
||||
case 'r':
|
||||
goto yy25
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy23:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'E':
|
||||
fallthrough
|
||||
case 'e':
|
||||
goto yy25
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy24:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'E':
|
||||
fallthrough
|
||||
case 'e':
|
||||
goto yy26
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy25:
|
||||
cursor++
|
||||
{
|
||||
return true
|
||||
}
|
||||
yy26:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'N':
|
||||
fallthrough
|
||||
case 'n':
|
||||
goto yy27
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy27:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'B':
|
||||
fallthrough
|
||||
case 'b':
|
||||
goto yy28
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy28:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'Y':
|
||||
fallthrough
|
||||
case 'y':
|
||||
goto yy25
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy29:
|
||||
{
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
/*!include:re2c "base.re" */
|
||||
|
||||
// Original pattern: (?i)byline|author|dateline|writtenby|p-author
|
||||
func IsByline(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
byline = byline|author|dateline|writtenby|p-author;
|
||||
|
||||
{byline} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
+2129
File diff suppressed because it is too large
Load Diff
+100
@@ -0,0 +1,100 @@
|
||||
/*!include:re2c "base.re" */
|
||||
|
||||
// Original pattern: (?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story
|
||||
func IsPositiveClass(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
positive = article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story;
|
||||
|
||||
{positive} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
// For IsNegativeClass, its original pattern is like this:
|
||||
// (?i)-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget
|
||||
//
|
||||
// Unfortunately, re2go doesn't handle anchor like ^ and $ internally, so for convenience
|
||||
// I'll split that pattern into two:
|
||||
// - `^hid$| hid$| hid |^hid `
|
||||
// - `-ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget`
|
||||
func IsNegativeClass(input string) bool {
|
||||
return isNegativeClass1(input) || isNegativeClass2(input)
|
||||
}
|
||||
|
||||
// This one handle: `^hid$| hid$| hid |^hid `
|
||||
func isNegativeClass1(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
// Variable for capturing parentheses (twice the number of groups).
|
||||
/*!maxnmatch:re2c*/
|
||||
yypmatch := make([]int, YYMAXNMATCH*2)
|
||||
var yynmatch int
|
||||
_ = yynmatch
|
||||
|
||||
// Autogenerated tag variables used by the lexer to track tag values.
|
||||
/*!stags:re2c format = 'var @@ int; _ = @@\n'; */
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:posix-captures = 1;
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
(.)?hid(.)? {
|
||||
// Extract submatch first
|
||||
var sub1, sub2 string
|
||||
start, end := yypmatch[0], yypmatch[1]
|
||||
|
||||
if yypmatch[2] != -1 {
|
||||
sub1 = input[yypmatch[2]:yypmatch[3]]
|
||||
}
|
||||
|
||||
if yypmatch[4] != -1 {
|
||||
sub2 = input[yypmatch[4]:yypmatch[5]]
|
||||
}
|
||||
|
||||
case1 := start == 0 && end == limit && sub1 == "" && sub2 == "" // '^hid$'
|
||||
case2 := end == limit && sub1 == " " && sub2 == "" // ' hid$'
|
||||
case3 := start == 0 && sub1 == "" && sub2 == " " // '^hid '
|
||||
case4 := sub1 == " " && sub2 == " " // ' hid '
|
||||
|
||||
if case1 || case2 || case3 || case4 {
|
||||
return true
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
// This one handle: `-ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget`
|
||||
func isNegativeClass2(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
negative = -ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget;
|
||||
|
||||
{negative} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
+2161
File diff suppressed because it is too large
Load Diff
+61
@@ -0,0 +1,61 @@
|
||||
/*!include:re2c "base.re" */
|
||||
|
||||
// Original pattern: (?i)-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote
|
||||
func IsUnlikelyCandidates(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
unlikely = -ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote;
|
||||
|
||||
{unlikely} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
// Original pattern: (?i)and|article|body|column|content|main|shadow
|
||||
func MaybeItsACandidate(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
maybe = and|article|body|column|content|main|shadow;
|
||||
|
||||
{maybe} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
// Commas as used in Latin, Sindhi, Chinese and various other scripts.
|
||||
// see: https://en.wikipedia.org/wiki/Comma#Comma_variants
|
||||
// Original pattern: \u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C
|
||||
func CountCommas(input string) int {
|
||||
var count int
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
commas = [\u002C\u060C\uFE50\uFE10\uFE11\u2E41\u2E34\u2E32\uFF0C];
|
||||
|
||||
{commas} { count++; continue }
|
||||
* { continue }
|
||||
$ { return count }
|
||||
*/
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
// Code generated by re2go 4.0.2, DO NOT EDIT.
|
||||
package re2go
|
||||
|
||||
import "strings"
|
||||
|
||||
// Original pattern: (?i)\s{2,}
|
||||
func NormalizeSpaces(input string) string {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
// Variable for capturing parentheses (twice the number of groups).
|
||||
const YYMAXNMATCH = 1
|
||||
|
||||
yypmatch := make([]int, YYMAXNMATCH*2)
|
||||
var yynmatch int
|
||||
_ = yynmatch
|
||||
|
||||
// Autogenerated tag variables used by the lexer to track tag values.
|
||||
var yyt1 int
|
||||
_ = yyt1
|
||||
|
||||
var start int
|
||||
var sb strings.Builder
|
||||
|
||||
for {
|
||||
{
|
||||
var yych byte
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case '\t', '\n':
|
||||
fallthrough
|
||||
case '\f', '\r':
|
||||
fallthrough
|
||||
case ' ':
|
||||
yyt1 = cursor
|
||||
goto yy3
|
||||
default:
|
||||
if limit <= cursor {
|
||||
goto yy6
|
||||
}
|
||||
goto yy1
|
||||
}
|
||||
yy1:
|
||||
cursor++
|
||||
yy2:
|
||||
{
|
||||
continue
|
||||
}
|
||||
yy3:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case '\t', '\n':
|
||||
fallthrough
|
||||
case '\f', '\r':
|
||||
fallthrough
|
||||
case ' ':
|
||||
goto yy4
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy4:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case '\t', '\n':
|
||||
fallthrough
|
||||
case '\f', '\r':
|
||||
fallthrough
|
||||
case ' ':
|
||||
goto yy4
|
||||
default:
|
||||
goto yy5
|
||||
}
|
||||
yy5:
|
||||
yynmatch = 1
|
||||
yypmatch[0] = yyt1
|
||||
yypmatch[1] = cursor
|
||||
{
|
||||
sb.WriteString(input[start:yypmatch[0]])
|
||||
sb.WriteString(" ")
|
||||
start = yypmatch[1]
|
||||
continue
|
||||
}
|
||||
yy6:
|
||||
{
|
||||
sb.WriteString(input[start:limit])
|
||||
return sb.String()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/*!include:re2c "base.re" */
|
||||
|
||||
import "strings"
|
||||
|
||||
// Original pattern: (?i)\s{2,}
|
||||
func NormalizeSpaces(input string) string {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
// Variable for capturing parentheses (twice the number of groups).
|
||||
/*!maxnmatch:re2c*/
|
||||
yypmatch := make([]int, YYMAXNMATCH*2)
|
||||
var yynmatch int
|
||||
_ = yynmatch
|
||||
|
||||
// Autogenerated tag variables used by the lexer to track tag values.
|
||||
/*!stags:re2c format = 'var @@ int; _ = @@\n'; */
|
||||
|
||||
var start int
|
||||
var sb strings.Builder
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:posix-captures = 1;
|
||||
|
||||
[\t\n\f\r ]{2,} {
|
||||
sb.WriteString(input[start:yypmatch[0]])
|
||||
sb.WriteString(" ")
|
||||
start = yypmatch[1]
|
||||
continue
|
||||
}
|
||||
|
||||
$ {
|
||||
sb.WriteString(input[start:limit])
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
* { continue }
|
||||
*/
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user