p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests
Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
test-pages/* linguist-vendored
|
||||
@@ -0,0 +1,6 @@
|
||||
.vscode/
|
||||
js/*
|
||||
|
||||
# Hidden script for development
|
||||
scripts/internal/
|
||||
.DS_Store
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
language: go
|
||||
sudo: true
|
||||
matrix:
|
||||
include:
|
||||
- go: 1.x
|
||||
env: LATEST=true
|
||||
- go: "1.13"
|
||||
|
||||
install:
|
||||
- go get github.com/go-shiori/go-readability
|
||||
|
||||
script:
|
||||
- go get -t -v ./...
|
||||
- diff -u <(echo -n) <(gofmt -d .)
|
||||
- go vet $(go list ./... | grep -v /vendor/)
|
||||
- go test -v -race ./...
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2019 Radhi Fadlillah
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
generate:
|
||||
@for name in internal/re2go/*.re; do \
|
||||
RE_IN=$$name; \
|
||||
RE_OUT=$$(echo $$name | sed 's/\.re/.go/'); \
|
||||
re2go -W -F --input-encoding utf8 --utf8 --no-generation-date -i $$RE_IN -o $$RE_OUT; \
|
||||
gofmt -w $$RE_OUT; \
|
||||
done
|
||||
|
||||
test: generate
|
||||
go test -timeout 30s ./...
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
# Go-Readability [![Go Reference][go-ref-badge]][go-ref] [![PayPal][paypal-badge]][paypal] [![Ko-fi][kofi-badge]][kofi]
|
||||
|
||||
Go-Readability is a Go package that find the main readable content and the metadata from a HTML page. It works by removing clutter like buttons, ads, background images, script, etc.
|
||||
|
||||
This package is based from [Readability.js] by [Mozilla] and written line by line to make sure it looks and works as similar as possible. This way, hopefully all web page that can be parsed by Readability.js are parse-able by go-readability as well.
|
||||
|
||||
> [!WARNING]
|
||||
> This package is deprecated in favor of [codeberg.org/readeck/go-readability/v2](https://codeberg.org/readeck/go-readability/src/branch/v2).
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Table of Contents](#table-of-contents)
|
||||
- [Status](#status)
|
||||
- [Installation](#installation)
|
||||
- [Example](#example)
|
||||
- [Command Line Usage](#command-line-usage)
|
||||
- [Licenses](#licenses)
|
||||
|
||||
## Status
|
||||
|
||||
This package is stable enough for use and up to date with [Readability.js v0.5.0][last-version].
|
||||
|
||||
For compatibility with Readability.js v0.6, use [codeberg.org/readeck/go-readability/v2](https://codeberg.org/readeck/go-readability/src/branch/v2).
|
||||
|
||||
## Installation
|
||||
|
||||
To install this package, just run `go get` :
|
||||
|
||||
```
|
||||
go get -u -v github.com/go-shiori/go-readability
|
||||
```
|
||||
|
||||
## Example
|
||||
|
||||
To get the readable content from an URL, you can use `readability.FromURL`. It will fetch the web page from specified url, check if it's readable, then parses the response to find the readable content :
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
readability "github.com/go-shiori/go-readability"
|
||||
)
|
||||
|
||||
var (
|
||||
urls = []string{
|
||||
// this one is article, so it's parse-able
|
||||
"https://www.nytimes.com/2019/02/20/climate/climate-national-security-threat.html",
|
||||
// while this one is not an article, so readability will fail to parse.
|
||||
"https://www.nytimes.com/",
|
||||
}
|
||||
)
|
||||
|
||||
func main() {
|
||||
for i, url := range urls {
|
||||
article, err := readability.FromURL(url, 30*time.Second)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to parse %s, %v\n", url, err)
|
||||
}
|
||||
|
||||
dstTxtFile, _ := os.Create(fmt.Sprintf("text-%02d.txt", i+1))
|
||||
defer dstTxtFile.Close()
|
||||
dstTxtFile.WriteString(article.TextContent)
|
||||
|
||||
dstHTMLFile, _ := os.Create(fmt.Sprintf("html-%02d.html", i+1))
|
||||
defer dstHTMLFile.Close()
|
||||
dstHTMLFile.WriteString(article.Content)
|
||||
|
||||
fmt.Printf("URL : %s\n", url)
|
||||
fmt.Printf("Title : %s\n", article.Title)
|
||||
fmt.Printf("Author : %s\n", article.Byline)
|
||||
fmt.Printf("Length : %d\n", article.Length)
|
||||
fmt.Printf("Excerpt : %s\n", article.Excerpt)
|
||||
fmt.Printf("SiteName: %s\n", article.SiteName)
|
||||
fmt.Printf("Image : %s\n", article.Image)
|
||||
fmt.Printf("Favicon : %s\n", article.Favicon)
|
||||
fmt.Printf("Text content saved to \"text-%02d.txt\"\n", i+1)
|
||||
fmt.Printf("HTML content saved to \"html-%02d.html\"\n", i+1)
|
||||
fmt.Println()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
However, sometimes you want to parse an URL no matter if it's an article or not. For example is when you only want to get metadata of the page. To do that, you have to download the page manually using `http.Get`, then parse it using `readability.FromReader` :
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
readability "github.com/go-shiori/go-readability"
|
||||
)
|
||||
|
||||
var (
|
||||
urls = []string{
|
||||
// Both will be parse-able now
|
||||
"https://www.nytimes.com/2019/02/20/climate/climate-national-security-threat.html",
|
||||
// But this one will not have any content
|
||||
"https://www.nytimes.com/",
|
||||
}
|
||||
)
|
||||
|
||||
func main() {
|
||||
for _, u := range urls {
|
||||
resp, err := http.Get(u)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to download %s: %v\n", u, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
parsedURL, err := url.Parse(u)
|
||||
if err != nil {
|
||||
log.Fatalf("error parsing url")
|
||||
}
|
||||
|
||||
article, err := readability.FromReader(resp.Body, parsedURL)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to parse %s: %v\n", u, err)
|
||||
}
|
||||
|
||||
fmt.Printf("URL : %s\n", u)
|
||||
fmt.Printf("Title : %s\n", article.Title)
|
||||
fmt.Printf("Author : %s\n", article.Byline)
|
||||
fmt.Printf("Length : %d\n", article.Length)
|
||||
fmt.Printf("Excerpt : %s\n", article.Excerpt)
|
||||
fmt.Printf("SiteName: %s\n", article.SiteName)
|
||||
fmt.Printf("Image : %s\n", article.Image)
|
||||
fmt.Printf("Favicon : %s\n", article.Favicon)
|
||||
fmt.Println()
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Command Line Usage
|
||||
|
||||
You can also use `go-readability` as command line app. To do that, first install the CLI :
|
||||
|
||||
```
|
||||
go install github.com/go-shiori/go-readability/cmd/go-readability@latest
|
||||
```
|
||||
|
||||
Now you can use it by running `go-readability` in your terminal :
|
||||
|
||||
```
|
||||
$ go-readability -h
|
||||
|
||||
go-readability is parser to fetch the readable content of a web page.
|
||||
The source can be an url or existing file in your storage.
|
||||
|
||||
Usage:
|
||||
go-readability [flags] source
|
||||
|
||||
Flags:
|
||||
-h, --help help for go-readability
|
||||
-l, --http string start the http server at the specified address
|
||||
-m, --metadata only print the page's metadata
|
||||
-t, --text only print the page's text
|
||||
```
|
||||
|
||||
## Licenses
|
||||
|
||||
Go-Readability is distributed under [MIT license][mit], which means you can use and modify it however you want. However, if you make an enhancement for it, if possible, please send a pull request. If you like this project, please consider donating to me either via [PayPal][paypal] or [Ko-Fi][kofi].
|
||||
|
||||
[go-ref]: https://pkg.go.dev/github.com/go-shiori/go-readability
|
||||
[go-ref-badge]: https://img.shields.io/static/v1?label=&message=Reference&color=007d9c&logo=go&logoColor=white
|
||||
[paypal]: https://www.paypal.me/RadhiFadlillah
|
||||
[paypal-badge]: https://img.shields.io/static/v1?label=&message=PayPal&color=00457C&logo=paypal&logoColor=white
|
||||
[kofi]: https://ko-fi.com/radhifadlillah
|
||||
[kofi-badge]: https://img.shields.io/static/v1?label=&message=Ko-fi&color=F16061&logo=ko-fi&logoColor=white
|
||||
[readability.js]: https://github.com/mozilla/readability
|
||||
[mozilla]: https://github.com/mozilla
|
||||
[last-version]: https://github.com/mozilla/readability/tree/0.5.0
|
||||
[mit]: https://choosealicense.com/licenses/mit/
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
// Code generated by re2go 4.0.2, DO NOT EDIT.
|
||||
package re2go
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package re2go
|
||||
|
||||
/*!rules:re2c:base_template
|
||||
re2c:eof = 0;
|
||||
re2c:yyfill:enable = 0;
|
||||
re2c:posix-captures = 0;
|
||||
re2c:case-insensitive = 0;
|
||||
|
||||
re2c:define:YYCTYPE = byte;
|
||||
re2c:define:YYPEEK = "input[cursor]";
|
||||
re2c:define:YYSKIP = "cursor++";
|
||||
re2c:define:YYBACKUP = "marker = cursor";
|
||||
re2c:define:YYRESTORE = "cursor = marker";
|
||||
re2c:define:YYLESSTHAN = "limit <= cursor";
|
||||
re2c:define:YYSTAGP = "@@{tag} = cursor";
|
||||
re2c:define:YYSTAGN = "@@{tag} = -1";
|
||||
re2c:define:YYSHIFTSTAG = "@@{tag} += @@{shift}";
|
||||
*/
|
||||
+330
@@ -0,0 +1,330 @@
|
||||
// Code generated by re2go 4.0.2, DO NOT EDIT.
|
||||
package re2go
|
||||
|
||||
// Original pattern: (?i)byline|author|dateline|writtenby|p-author
|
||||
func IsByline(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for {
|
||||
{
|
||||
var yych byte
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'A':
|
||||
fallthrough
|
||||
case 'a':
|
||||
goto yy3
|
||||
case 'B':
|
||||
fallthrough
|
||||
case 'b':
|
||||
goto yy4
|
||||
case 'D':
|
||||
fallthrough
|
||||
case 'd':
|
||||
goto yy5
|
||||
case 'P':
|
||||
fallthrough
|
||||
case 'p':
|
||||
goto yy6
|
||||
case 'W':
|
||||
fallthrough
|
||||
case 'w':
|
||||
goto yy7
|
||||
default:
|
||||
if limit <= cursor {
|
||||
goto yy29
|
||||
}
|
||||
goto yy1
|
||||
}
|
||||
yy1:
|
||||
cursor++
|
||||
yy2:
|
||||
{
|
||||
continue
|
||||
}
|
||||
yy3:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'U':
|
||||
fallthrough
|
||||
case 'u':
|
||||
goto yy8
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy4:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'Y':
|
||||
fallthrough
|
||||
case 'y':
|
||||
goto yy10
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy5:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'A':
|
||||
fallthrough
|
||||
case 'a':
|
||||
goto yy11
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy6:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case '-':
|
||||
goto yy12
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy7:
|
||||
cursor++
|
||||
marker = cursor
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'R':
|
||||
fallthrough
|
||||
case 'r':
|
||||
goto yy13
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy8:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'T':
|
||||
fallthrough
|
||||
case 't':
|
||||
goto yy14
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy9:
|
||||
cursor = marker
|
||||
goto yy2
|
||||
yy10:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'L':
|
||||
fallthrough
|
||||
case 'l':
|
||||
goto yy15
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy11:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'T':
|
||||
fallthrough
|
||||
case 't':
|
||||
goto yy16
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy12:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'A':
|
||||
fallthrough
|
||||
case 'a':
|
||||
goto yy17
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy13:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'I':
|
||||
fallthrough
|
||||
case 'i':
|
||||
goto yy18
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy14:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'H':
|
||||
fallthrough
|
||||
case 'h':
|
||||
goto yy19
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy15:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'I':
|
||||
fallthrough
|
||||
case 'i':
|
||||
goto yy20
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy16:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'E':
|
||||
fallthrough
|
||||
case 'e':
|
||||
goto yy10
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy17:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'U':
|
||||
fallthrough
|
||||
case 'u':
|
||||
goto yy8
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy18:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'T':
|
||||
fallthrough
|
||||
case 't':
|
||||
goto yy21
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy19:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'O':
|
||||
fallthrough
|
||||
case 'o':
|
||||
goto yy22
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy20:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'N':
|
||||
fallthrough
|
||||
case 'n':
|
||||
goto yy23
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy21:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'T':
|
||||
fallthrough
|
||||
case 't':
|
||||
goto yy24
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy22:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'R':
|
||||
fallthrough
|
||||
case 'r':
|
||||
goto yy25
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy23:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'E':
|
||||
fallthrough
|
||||
case 'e':
|
||||
goto yy25
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy24:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'E':
|
||||
fallthrough
|
||||
case 'e':
|
||||
goto yy26
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy25:
|
||||
cursor++
|
||||
{
|
||||
return true
|
||||
}
|
||||
yy26:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'N':
|
||||
fallthrough
|
||||
case 'n':
|
||||
goto yy27
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy27:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'B':
|
||||
fallthrough
|
||||
case 'b':
|
||||
goto yy28
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy28:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case 'Y':
|
||||
fallthrough
|
||||
case 'y':
|
||||
goto yy25
|
||||
default:
|
||||
goto yy9
|
||||
}
|
||||
yy29:
|
||||
{
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
/*!include:re2c "base.re" */
|
||||
|
||||
// Original pattern: (?i)byline|author|dateline|writtenby|p-author
|
||||
func IsByline(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
byline = byline|author|dateline|writtenby|p-author;
|
||||
|
||||
{byline} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
+2129
File diff suppressed because it is too large
Load Diff
+100
@@ -0,0 +1,100 @@
|
||||
/*!include:re2c "base.re" */
|
||||
|
||||
// Original pattern: (?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story
|
||||
func IsPositiveClass(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
positive = article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story;
|
||||
|
||||
{positive} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
// For IsNegativeClass, its original pattern is like this:
|
||||
// (?i)-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget
|
||||
//
|
||||
// Unfortunately, re2go doesn't handle anchor like ^ and $ internally, so for convenience
|
||||
// I'll split that pattern into two:
|
||||
// - `^hid$| hid$| hid |^hid `
|
||||
// - `-ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget`
|
||||
func IsNegativeClass(input string) bool {
|
||||
return isNegativeClass1(input) || isNegativeClass2(input)
|
||||
}
|
||||
|
||||
// This one handle: `^hid$| hid$| hid |^hid `
|
||||
func isNegativeClass1(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
// Variable for capturing parentheses (twice the number of groups).
|
||||
/*!maxnmatch:re2c*/
|
||||
yypmatch := make([]int, YYMAXNMATCH*2)
|
||||
var yynmatch int
|
||||
_ = yynmatch
|
||||
|
||||
// Autogenerated tag variables used by the lexer to track tag values.
|
||||
/*!stags:re2c format = 'var @@ int; _ = @@\n'; */
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:posix-captures = 1;
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
(.)?hid(.)? {
|
||||
// Extract submatch first
|
||||
var sub1, sub2 string
|
||||
start, end := yypmatch[0], yypmatch[1]
|
||||
|
||||
if yypmatch[2] != -1 {
|
||||
sub1 = input[yypmatch[2]:yypmatch[3]]
|
||||
}
|
||||
|
||||
if yypmatch[4] != -1 {
|
||||
sub2 = input[yypmatch[4]:yypmatch[5]]
|
||||
}
|
||||
|
||||
case1 := start == 0 && end == limit && sub1 == "" && sub2 == "" // '^hid$'
|
||||
case2 := end == limit && sub1 == " " && sub2 == "" // ' hid$'
|
||||
case3 := start == 0 && sub1 == "" && sub2 == " " // '^hid '
|
||||
case4 := sub1 == " " && sub2 == " " // ' hid '
|
||||
|
||||
if case1 || case2 || case3 || case4 {
|
||||
return true
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
// This one handle: `-ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget`
|
||||
func isNegativeClass2(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
negative = -ad-|hidden|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget;
|
||||
|
||||
{negative} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
+2161
File diff suppressed because it is too large
Load Diff
+61
@@ -0,0 +1,61 @@
|
||||
/*!include:re2c "base.re" */
|
||||
|
||||
// Original pattern: (?i)-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote
|
||||
func IsUnlikelyCandidates(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
unlikely = -ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote;
|
||||
|
||||
{unlikely} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
// Original pattern: (?i)and|article|body|column|content|main|shadow
|
||||
func MaybeItsACandidate(input string) bool {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
maybe = and|article|body|column|content|main|shadow;
|
||||
|
||||
{maybe} { return true }
|
||||
* { continue }
|
||||
$ { return false }
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
// Commas as used in Latin, Sindhi, Chinese and various other scripts.
|
||||
// see: https://en.wikipedia.org/wiki/Comma#Comma_variants
|
||||
// Original pattern: \u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C
|
||||
func CountCommas(input string) int {
|
||||
var count int
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:case-insensitive = 1;
|
||||
|
||||
commas = [\u002C\u060C\uFE50\uFE10\uFE11\u2E41\u2E34\u2E32\uFF0C];
|
||||
|
||||
{commas} { count++; continue }
|
||||
* { continue }
|
||||
$ { return count }
|
||||
*/
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
// Code generated by re2go 4.0.2, DO NOT EDIT.
|
||||
package re2go
|
||||
|
||||
import "strings"
|
||||
|
||||
// Original pattern: (?i)\s{2,}
|
||||
func NormalizeSpaces(input string) string {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
// Variable for capturing parentheses (twice the number of groups).
|
||||
const YYMAXNMATCH = 1
|
||||
|
||||
yypmatch := make([]int, YYMAXNMATCH*2)
|
||||
var yynmatch int
|
||||
_ = yynmatch
|
||||
|
||||
// Autogenerated tag variables used by the lexer to track tag values.
|
||||
var yyt1 int
|
||||
_ = yyt1
|
||||
|
||||
var start int
|
||||
var sb strings.Builder
|
||||
|
||||
for {
|
||||
{
|
||||
var yych byte
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case '\t', '\n':
|
||||
fallthrough
|
||||
case '\f', '\r':
|
||||
fallthrough
|
||||
case ' ':
|
||||
yyt1 = cursor
|
||||
goto yy3
|
||||
default:
|
||||
if limit <= cursor {
|
||||
goto yy6
|
||||
}
|
||||
goto yy1
|
||||
}
|
||||
yy1:
|
||||
cursor++
|
||||
yy2:
|
||||
{
|
||||
continue
|
||||
}
|
||||
yy3:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case '\t', '\n':
|
||||
fallthrough
|
||||
case '\f', '\r':
|
||||
fallthrough
|
||||
case ' ':
|
||||
goto yy4
|
||||
default:
|
||||
goto yy2
|
||||
}
|
||||
yy4:
|
||||
cursor++
|
||||
yych = input[cursor]
|
||||
switch yych {
|
||||
case '\t', '\n':
|
||||
fallthrough
|
||||
case '\f', '\r':
|
||||
fallthrough
|
||||
case ' ':
|
||||
goto yy4
|
||||
default:
|
||||
goto yy5
|
||||
}
|
||||
yy5:
|
||||
yynmatch = 1
|
||||
yypmatch[0] = yyt1
|
||||
yypmatch[1] = cursor
|
||||
{
|
||||
sb.WriteString(input[start:yypmatch[0]])
|
||||
sb.WriteString(" ")
|
||||
start = yypmatch[1]
|
||||
continue
|
||||
}
|
||||
yy6:
|
||||
{
|
||||
sb.WriteString(input[start:limit])
|
||||
return sb.String()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/*!include:re2c "base.re" */
|
||||
|
||||
import "strings"
|
||||
|
||||
// Original pattern: (?i)\s{2,}
|
||||
func NormalizeSpaces(input string) string {
|
||||
var cursor, marker int
|
||||
input += string(rune(0)) // add terminating null
|
||||
limit := len(input) - 1 // limit points at the terminating null
|
||||
_ = marker
|
||||
|
||||
// Variable for capturing parentheses (twice the number of groups).
|
||||
/*!maxnmatch:re2c*/
|
||||
yypmatch := make([]int, YYMAXNMATCH*2)
|
||||
var yynmatch int
|
||||
_ = yynmatch
|
||||
|
||||
// Autogenerated tag variables used by the lexer to track tag values.
|
||||
/*!stags:re2c format = 'var @@ int; _ = @@\n'; */
|
||||
|
||||
var start int
|
||||
var sb strings.Builder
|
||||
|
||||
for { /*!use:re2c:base_template
|
||||
re2c:posix-captures = 1;
|
||||
|
||||
[\t\n\f\r ]{2,} {
|
||||
sb.WriteString(input[start:yypmatch[0]])
|
||||
sb.WriteString(" ")
|
||||
start = yypmatch[1]
|
||||
continue
|
||||
}
|
||||
|
||||
$ {
|
||||
sb.WriteString(input[start:limit])
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
* { continue }
|
||||
*/
|
||||
}
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
package readability
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/go-shiori/dom"
|
||||
"github.com/go-shiori/go-readability/internal/re2go"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// Check checks whether the input is readable without parsing the whole thing.
|
||||
func (ps *Parser) Check(input io.Reader) bool {
|
||||
// Parse input
|
||||
doc, err := dom.Parse(input)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return ps.CheckDocument(doc)
|
||||
}
|
||||
|
||||
// CheckDocument checks whether the document is readable without parsing the whole thing.
|
||||
func (ps *Parser) CheckDocument(doc *html.Node) bool {
|
||||
// Get <p> and <pre> nodes.
|
||||
nodes := dom.QuerySelectorAll(doc, "p, pre, article")
|
||||
|
||||
// Also get <div> nodes which have <br> node(s) and append
|
||||
// them into the `nodes` variable.
|
||||
// Some articles' DOM structures might look like :
|
||||
//
|
||||
// <div>
|
||||
// Sentences<br>
|
||||
// <br>
|
||||
// Sentences<br>
|
||||
// </div>
|
||||
//
|
||||
// So we need to make sure only fetch the div once.
|
||||
// To do so, we will use map as dictionary.
|
||||
tracker := make(map[*html.Node]struct{})
|
||||
for _, br := range dom.QuerySelectorAll(doc, "div > br") {
|
||||
if br.Parent == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, exist := tracker[br.Parent]; !exist {
|
||||
tracker[br.Parent] = struct{}{}
|
||||
nodes = append(nodes, br.Parent)
|
||||
}
|
||||
}
|
||||
|
||||
// This is a little cheeky, we use the accumulator 'score' to decide what
|
||||
// to return from this callback.
|
||||
score := float64(0)
|
||||
return ps.someNode(nodes, func(node *html.Node) bool {
|
||||
if !ps.isProbablyVisible(node) {
|
||||
return false
|
||||
}
|
||||
|
||||
matchString := dom.ClassName(node) + " " + dom.ID(node)
|
||||
if re2go.IsUnlikelyCandidates(matchString) &&
|
||||
!re2go.MaybeItsACandidate(matchString) {
|
||||
return false
|
||||
}
|
||||
|
||||
if dom.TagName(node) == "p" && ps.hasAncestorTag(node, "li", -1, nil) {
|
||||
return false
|
||||
}
|
||||
|
||||
nodeText := strings.TrimSpace(dom.TextContent(node))
|
||||
nodeTextLength := len(nodeText)
|
||||
if nodeTextLength < 140 {
|
||||
return false
|
||||
}
|
||||
|
||||
score += math.Sqrt(float64(nodeTextLength - 140))
|
||||
return score > 20
|
||||
})
|
||||
}
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
package readability
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
nurl "net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/araddon/dateparse"
|
||||
"github.com/go-shiori/dom"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// Parse parses a reader and find the main readable content.
|
||||
func (ps *Parser) Parse(input io.Reader, pageURL *nurl.URL) (Article, error) {
|
||||
// Parse input
|
||||
doc, err := dom.Parse(input)
|
||||
if err != nil {
|
||||
return Article{}, fmt.Errorf("failed to parse input: %v", err)
|
||||
}
|
||||
|
||||
return ps.ParseDocument(doc, pageURL)
|
||||
}
|
||||
|
||||
// ParseDocument parses the specified document and find the main readable content.
|
||||
func (ps *Parser) ParseDocument(doc *html.Node, pageURL *nurl.URL) (Article, error) {
|
||||
// Clone document to make sure the original kept untouched
|
||||
ps.doc = dom.Clone(doc, true)
|
||||
|
||||
// Reset parser data
|
||||
ps.articleTitle = ""
|
||||
ps.articleByline = ""
|
||||
ps.articleDir = ""
|
||||
ps.articleSiteName = ""
|
||||
ps.documentURI = pageURL
|
||||
ps.attempts = []parseAttempt{}
|
||||
ps.flags = flags{
|
||||
stripUnlikelys: true,
|
||||
useWeightClasses: true,
|
||||
cleanConditionally: true,
|
||||
}
|
||||
|
||||
// Avoid parsing too large documents, as per configuration option
|
||||
if ps.MaxElemsToParse > 0 {
|
||||
numTags := len(dom.GetElementsByTagName(ps.doc, "*"))
|
||||
if numTags > ps.MaxElemsToParse {
|
||||
return Article{}, fmt.Errorf("documents too large: %d elements", numTags)
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrap image from noscript
|
||||
ps.unwrapNoscriptImages(ps.doc)
|
||||
|
||||
// Extract JSON-LD metadata before removing scripts
|
||||
var jsonLd map[string]string
|
||||
if !ps.DisableJSONLD {
|
||||
jsonLd, _ = ps.getJSONLD()
|
||||
}
|
||||
|
||||
// Remove script tags from the document.
|
||||
ps.removeScripts(ps.doc)
|
||||
|
||||
// Prepares the HTML document
|
||||
ps.prepDocument()
|
||||
|
||||
// Fetch metadata
|
||||
metadata := ps.getArticleMetadata(jsonLd)
|
||||
ps.articleTitle = metadata["title"]
|
||||
|
||||
// Try to grab article content
|
||||
finalHTMLContent := ""
|
||||
finalTextContent := ""
|
||||
articleContent := ps.grabArticle()
|
||||
var readableNode *html.Node
|
||||
|
||||
if articleContent != nil {
|
||||
ps.postProcessContent(articleContent)
|
||||
|
||||
// If we haven't found an excerpt in the article's metadata,
|
||||
// use the article's first paragraph as the excerpt. This is used
|
||||
// for displaying a preview of the article's content.
|
||||
if metadata["excerpt"] == "" {
|
||||
paragraphs := dom.GetElementsByTagName(articleContent, "p")
|
||||
if len(paragraphs) > 0 {
|
||||
metadata["excerpt"] = strings.TrimSpace(dom.TextContent(paragraphs[0]))
|
||||
}
|
||||
}
|
||||
|
||||
readableNode = dom.FirstElementChild(articleContent)
|
||||
finalHTMLContent = dom.InnerHTML(articleContent)
|
||||
finalTextContent = dom.TextContent(articleContent)
|
||||
finalTextContent = strings.TrimSpace(finalTextContent)
|
||||
}
|
||||
|
||||
finalByline := metadata["byline"]
|
||||
if finalByline == "" {
|
||||
finalByline = ps.articleByline
|
||||
}
|
||||
|
||||
// Excerpt is an supposed to be short and concise,
|
||||
// so it shouldn't have any new line
|
||||
excerpt := strings.TrimSpace(metadata["excerpt"])
|
||||
excerpt = strings.Join(strings.Fields(excerpt), " ")
|
||||
|
||||
// go-readability special:
|
||||
// Internet is dangerous and weird, and sometimes we will find
|
||||
// metadata isn't encoded using a valid Utf-8, so here we check it.
|
||||
var replacementTitle string
|
||||
if pageURL != nil {
|
||||
replacementTitle = pageURL.String()
|
||||
}
|
||||
|
||||
validTitle := strings.ToValidUTF8(ps.articleTitle, replacementTitle)
|
||||
validByline := strings.ToValidUTF8(finalByline, "")
|
||||
validExcerpt := strings.ToValidUTF8(excerpt, "")
|
||||
|
||||
publishedTime := ps.getDate(metadata, "publishedTime")
|
||||
modifiedTime := ps.getDate(metadata, "modifiedTime")
|
||||
|
||||
return Article{
|
||||
Title: validTitle,
|
||||
Byline: validByline,
|
||||
Node: readableNode,
|
||||
Content: finalHTMLContent,
|
||||
TextContent: finalTextContent,
|
||||
Length: charCount(finalTextContent),
|
||||
Excerpt: validExcerpt,
|
||||
SiteName: metadata["siteName"],
|
||||
Image: metadata["image"],
|
||||
Favicon: metadata["favicon"],
|
||||
Language: ps.articleLang,
|
||||
PublishedTime: publishedTime,
|
||||
ModifiedTime: modifiedTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getDate tries to get a date from metadata, and parse it using a list of known formats.
|
||||
func (ps *Parser) getDate(metadata map[string]string, fieldName string) *time.Time {
|
||||
dateStr, ok := metadata[fieldName]
|
||||
if ok && len(dateStr) > 0 {
|
||||
return ps.getParsedDate(dateStr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getParsedDate tries to parse a date string using a list of known formats.
|
||||
// If the date string can't be parsed, it will return nil.
|
||||
func (ps *Parser) getParsedDate(dateStr string) *time.Time {
|
||||
d, err := dateparse.ParseAny(dateStr)
|
||||
if err != nil {
|
||||
ps.logf("failed to parse date \"%s\": %v\n", dateStr, err)
|
||||
return nil
|
||||
}
|
||||
return &d
|
||||
}
|
||||
+2327
File diff suppressed because it is too large
Load Diff
+86
@@ -0,0 +1,86 @@
|
||||
// Package readability is a Go package that find the main readable
|
||||
// content from a HTML page. It works by removing clutter like buttons,
|
||||
// ads, background images, script, etc.
|
||||
//
|
||||
// This package is based from Readability.js by Mozilla, and written line
|
||||
// by line to make sure it looks and works as similar as possible. This
|
||||
// way, hopefully all web page that can be parsed by Readability.js
|
||||
// are parse-able by go-readability as well.
|
||||
package readability
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
nurl "net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// FromReader parses an `io.Reader` and returns the readable content. It's the wrapper
|
||||
// or `Parser.Parse()` and useful if you only want to use the default parser.
|
||||
func FromReader(input io.Reader, pageURL *nurl.URL) (Article, error) {
|
||||
parser := NewParser()
|
||||
return parser.Parse(input, pageURL)
|
||||
}
|
||||
|
||||
// FromDocument parses an document and returns the readable content. It's the wrapper
|
||||
// or `Parser.ParseDocument()` and useful if you only want to use the default parser.
|
||||
func FromDocument(doc *html.Node, pageURL *nurl.URL) (Article, error) {
|
||||
parser := NewParser()
|
||||
return parser.ParseDocument(doc, pageURL)
|
||||
}
|
||||
|
||||
type RequestWith func(r *http.Request)
|
||||
|
||||
// FromURL fetch the web page from specified url then parses the response to find
|
||||
// the readable content.
|
||||
func FromURL(pageURL string, timeout time.Duration, requestModifiers ...RequestWith) (Article, error) {
|
||||
// Make sure URL is valid
|
||||
parsedURL, err := nurl.ParseRequestURI(pageURL)
|
||||
if err != nil {
|
||||
return Article{}, fmt.Errorf("failed to parse URL: %v", err)
|
||||
}
|
||||
|
||||
// Fetch page from URL
|
||||
client := &http.Client{Timeout: timeout}
|
||||
req, err := http.NewRequest("GET", pageURL, nil)
|
||||
for _, modifer := range requestModifiers {
|
||||
modifer(req)
|
||||
}
|
||||
if err != nil {
|
||||
return Article{}, fmt.Errorf("failed to fetch the page: %v", err)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return Article{}, fmt.Errorf("failed to fetch the page: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Make sure content type is HTML
|
||||
cp := resp.Header.Get("Content-Type")
|
||||
if !strings.Contains(cp, "text/html") {
|
||||
return Article{}, fmt.Errorf("URL is not a HTML document")
|
||||
}
|
||||
|
||||
// Parse content
|
||||
parser := NewParser()
|
||||
return parser.Parse(resp.Body, parsedURL)
|
||||
}
|
||||
|
||||
// Check checks whether the input is readable without parsing the whole thing. It's the
|
||||
// wrapper for `Parser.Check()` and useful if you only use the default parser.
|
||||
func Check(input io.Reader) bool {
|
||||
parser := NewParser()
|
||||
return parser.Check(input)
|
||||
}
|
||||
|
||||
// CheckDocument checks whether the document is readable without parsing the whole thing.
|
||||
// It's the wrapper for `Parser.CheckDocument()` and useful if you only use the default
|
||||
// parser.
|
||||
func CheckDocument(doc *html.Node) bool {
|
||||
parser := NewParser()
|
||||
return parser.CheckDocument(doc)
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package readability
|
||||
|
||||
import (
|
||||
nurl "net/url"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// indexOf returns the position of the first occurrence of a
|
||||
// specified value in a string array. Returns -1 if the
|
||||
// value to search for never occurs.
|
||||
func indexOf(array []string, key string) int {
|
||||
for i := 0; i < len(array); i++ {
|
||||
if array[i] == key {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// wordCount returns number of word in str.
|
||||
func wordCount(str string) int {
|
||||
return len(strings.Fields(str))
|
||||
}
|
||||
|
||||
// charCount returns number of char in str.
|
||||
func charCount(str string) int {
|
||||
return utf8.RuneCountInString(str)
|
||||
}
|
||||
|
||||
// isValidURL checks if URL is valid.
|
||||
func isValidURL(s string) bool {
|
||||
_, err := nurl.ParseRequestURI(s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// toAbsoluteURI convert uri to absolute path based on base.
|
||||
// However, if uri is prefixed with hash (#), the uri won't be changed.
|
||||
func toAbsoluteURI(uri string, base *nurl.URL) string {
|
||||
if uri == "" || base == nil {
|
||||
return uri
|
||||
}
|
||||
|
||||
// If it is hash tag, return as it is
|
||||
if strings.HasPrefix(uri, "#") {
|
||||
return uri
|
||||
}
|
||||
|
||||
// If it is data URI, return as it is
|
||||
if strings.HasPrefix(uri, "data:") {
|
||||
return uri
|
||||
}
|
||||
|
||||
// If it is already an absolute URL, return as it is
|
||||
tmp, err := nurl.ParseRequestURI(uri)
|
||||
if err == nil && tmp.Scheme != "" && tmp.Hostname() != "" {
|
||||
return uri
|
||||
}
|
||||
|
||||
// Otherwise, resolve against base URI.
|
||||
tmp, err = nurl.Parse(uri)
|
||||
if err != nil {
|
||||
return uri
|
||||
}
|
||||
|
||||
return base.ResolveReference(tmp).String()
|
||||
}
|
||||
|
||||
// strOr returns the first not empty string in args.
|
||||
func strOr(args ...string) string {
|
||||
for i := 0; i < len(args); i++ {
|
||||
if args[i] != "" {
|
||||
return args[i]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sliceToMap(strings ...string) map[string]struct{} {
|
||||
result := make(map[string]struct{})
|
||||
for _, s := range strings {
|
||||
result[s] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func strFilter(strs []string, filter func(string) bool) []string {
|
||||
var result []string
|
||||
for _, s := range strs {
|
||||
if filter(s) {
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func trim(s string) string {
|
||||
s = strings.Join(strings.Fields(s), " ")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
Reference in New Issue
Block a user