kyle
9444a35322
Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
87 lines
2.7 KiB
Go
87 lines
2.7 KiB
Go
// Package readability is a Go package that find the main readable
|
|
// content from a HTML page. It works by removing clutter like buttons,
|
|
// ads, background images, script, etc.
|
|
//
|
|
// This package is based from Readability.js by Mozilla, and written line
|
|
// by line to make sure it looks and works as similar as possible. This
|
|
// way, hopefully all web page that can be parsed by Readability.js
|
|
// are parse-able by go-readability as well.
|
|
package readability
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
nurl "net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
// FromReader parses an `io.Reader` and returns the readable content. It's the wrapper
|
|
// or `Parser.Parse()` and useful if you only want to use the default parser.
|
|
func FromReader(input io.Reader, pageURL *nurl.URL) (Article, error) {
|
|
parser := NewParser()
|
|
return parser.Parse(input, pageURL)
|
|
}
|
|
|
|
// FromDocument parses an document and returns the readable content. It's the wrapper
|
|
// or `Parser.ParseDocument()` and useful if you only want to use the default parser.
|
|
func FromDocument(doc *html.Node, pageURL *nurl.URL) (Article, error) {
|
|
parser := NewParser()
|
|
return parser.ParseDocument(doc, pageURL)
|
|
}
|
|
|
|
type RequestWith func(r *http.Request)
|
|
|
|
// FromURL fetch the web page from specified url then parses the response to find
|
|
// the readable content.
|
|
func FromURL(pageURL string, timeout time.Duration, requestModifiers ...RequestWith) (Article, error) {
|
|
// Make sure URL is valid
|
|
parsedURL, err := nurl.ParseRequestURI(pageURL)
|
|
if err != nil {
|
|
return Article{}, fmt.Errorf("failed to parse URL: %v", err)
|
|
}
|
|
|
|
// Fetch page from URL
|
|
client := &http.Client{Timeout: timeout}
|
|
req, err := http.NewRequest("GET", pageURL, nil)
|
|
for _, modifer := range requestModifiers {
|
|
modifer(req)
|
|
}
|
|
if err != nil {
|
|
return Article{}, fmt.Errorf("failed to fetch the page: %v", err)
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return Article{}, fmt.Errorf("failed to fetch the page: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Make sure content type is HTML
|
|
cp := resp.Header.Get("Content-Type")
|
|
if !strings.Contains(cp, "text/html") {
|
|
return Article{}, fmt.Errorf("URL is not a HTML document")
|
|
}
|
|
|
|
// Parse content
|
|
parser := NewParser()
|
|
return parser.Parse(resp.Body, parsedURL)
|
|
}
|
|
|
|
// Check checks whether the input is readable without parsing the whole thing. It's the
|
|
// wrapper for `Parser.Check()` and useful if you only use the default parser.
|
|
func Check(input io.Reader) bool {
|
|
parser := NewParser()
|
|
return parser.Check(input)
|
|
}
|
|
|
|
// CheckDocument checks whether the document is readable without parsing the whole thing.
|
|
// It's the wrapper for `Parser.CheckDocument()` and useful if you only use the default
|
|
// parser.
|
|
func CheckDocument(doc *html.Node) bool {
|
|
parser := NewParser()
|
|
return parser.CheckDocument(doc)
|
|
}
|