package readability import ( "io" "math" "strings" "github.com/go-shiori/dom" "github.com/go-shiori/go-readability/internal/re2go" "golang.org/x/net/html" ) // Check checks whether the input is readable without parsing the whole thing. func (ps *Parser) Check(input io.Reader) bool { // Parse input doc, err := dom.Parse(input) if err != nil { return false } return ps.CheckDocument(doc) } // CheckDocument checks whether the document is readable without parsing the whole thing. func (ps *Parser) CheckDocument(doc *html.Node) bool { // Get
and
nodes. nodes := dom.QuerySelectorAll(doc, "p, pre, article") // Also getnodes which have
node(s) and append // them into the `nodes` variable. // Some articles' DOM structures might look like : // //// Sentences// // So we need to make sure only fetch the div once. // To do so, we will use map as dictionary. tracker := make(map[*html.Node]struct{}) for _, br := range dom.QuerySelectorAll(doc, "div > br") { if br.Parent == nil { continue } if _, exist := tracker[br.Parent]; !exist { tracker[br.Parent] = struct{}{} nodes = append(nodes, br.Parent) } } // This is a little cheeky, we use the accumulator 'score' to decide what // to return from this callback. score := float64(0) return ps.someNode(nodes, func(node *html.Node) bool { if !ps.isProbablyVisible(node) { return false } matchString := dom.ClassName(node) + " " + dom.ID(node) if re2go.IsUnlikelyCandidates(matchString) && !re2go.MaybeItsACandidate(matchString) { return false } if dom.TagName(node) == "p" && ps.hasAncestorTag(node, "li", -1, nil) { return false } nodeText := strings.TrimSpace(dom.TextContent(node)) nodeTextLength := len(nodeText) if nodeTextLength < 140 { return false } score += math.Sqrt(float64(nodeTextLength - 140)) return score > 20 }) }
//
// Sentences
//