p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification
is ignored. No 502. keys add writes the secret only to --out. Release binary
has no -test-allow-prefix.

internal/fetch/fetch.go

@@ -0,0 +1,203 @@
+package fetch
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/guard"
+)
+
+const (
+	DefaultMaxBody = 5 << 20
+	DefaultTimeout = 20 * time.Second
+)
+
+// Resource is one successful GET.
+type Resource struct {
+	URL         string
+	FinalURL    string
+	Status      int
+	ContentType string
+	Body        []byte
+	FetchedAt   time.Time
+	Hash        string
+}
+
+func (r *Resource) SHA256() string {
+	if r.Hash != "" {
+		return r.Hash
+	}
+	sum := sha256.Sum256(r.Body)
+	r.Hash = "sha256:" + hex.EncodeToString(sum[:])
+	return r.Hash
+}
+
+// Error is a fetch failure with the HTTP status sift should return to its caller.
+type Error struct {
+	Status int
+	Msg    string
+	Hint   string
+}
+
+func (e *Error) Error() string { return e.Msg }
+
+func errf(status int, hint, format string, args ...any) *Error {
+	return &Error{Status: status, Hint: hint, Msg: fmt.Sprintf(format, args...)}
+}
+
+// Fetcher is the only outbound HTTP path. Guard must be the same instance
+// that holds DenyPrefixes.
+type Fetcher struct {
+	Guard   *guard.Guard
+	UA      string
+	MaxBody int64
+	Timeout time.Duration
+}
+
+func (f *Fetcher) maxBody() int64 {
+	if f.MaxBody <= 0 {
+		return DefaultMaxBody
+	}
+	return f.MaxBody
+}
+
+func (f *Fetcher) timeout() time.Duration {
+	if f.Timeout <= 0 {
+		return DefaultTimeout
+	}
+	return f.Timeout
+}
+
+func (f *Fetcher) ua() string {
+	if f.UA != "" {
+		return f.UA
+	}
+	return "sift/dev"
+}
+
+// Fetch GETs rawURL after Guard.CheckURL, using Guard.Client() from the same instance.
+func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
+	if f == nil || f.Guard == nil {
+		return nil, errf(500, "", "internal error")
+	}
+	ctx, cancel := context.WithTimeout(ctx, f.timeout())
+	defer cancel()
+
+	if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil {
+		return nil, mapGuard(err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, errf(400, "", "invalid url")
+	}
+	req.Header.Set("User-Agent", f.ua())
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
+	// no cookies, no auth, no Referer
+
+	client := f.Guard.Client()
+	client.Timeout = f.timeout()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, mapDial(err)
+	}
+	defer resp.Body.Close()
+
+	limit := f.maxBody()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
+	if err != nil {
+		return nil, mapDial(err)
+	}
+	if int64(len(body)) > limit {
+		return nil, errf(413, "", "too large (>%d MB decompressed)", limit>>20)
+	}
+
+	final := rawURL
+	if resp.Request != nil && resp.Request.URL != nil {
+		final = resp.Request.URL.String()
+	}
+	ct := resp.Header.Get("Content-Type")
+	if i := strings.IndexByte(ct, ';'); i >= 0 {
+		// keep full header for charset; also store raw
+	}
+	res := &Resource{
+		URL:         rawURL,
+		FinalURL:    final,
+		Status:      resp.StatusCode,
+		ContentType: ct,
+		Body:        body,
+		FetchedAt:   time.Now().UTC(),
+	}
+
+	switch {
+	case resp.StatusCode == http.StatusOK:
+		return res, nil
+	case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451:
+		return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode)
+	case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header):
+		return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser")
+	case resp.StatusCode >= 400:
+		return res, errf(422, "", "upstream %d", resp.StatusCode)
+	default:
+		return res, nil
+	}
+}
+
+func looksLikeChallenge(body []byte, h http.Header) bool {
+	s := strings.ToLower(string(body))
+	if strings.Contains(s, "captcha") || strings.Contains(s, "cf-challenge") || strings.Contains(s, "just a moment") {
+		return true
+	}
+	if strings.Contains(strings.ToLower(h.Get("Server")), "cloudflare") {
+		return true
+	}
+	return false
+}
+
+func mapGuard(err error) error {
+	var b *guard.BlockedError
+	if errors.As(err, &b) {
+		return &Error{Status: 403, Msg: b.Error()}
+	}
+	if strings.HasPrefix(err.Error(), "blocked:") {
+		return &Error{Status: 403, Msg: err.Error()}
+	}
+	return &Error{Status: 403, Msg: "blocked: " + err.Error()}
+}
+
+func mapDial(err error) error {
+	if err == nil {
+		return nil
+	}
+	var b *guard.BlockedError
+	if errors.As(err, &b) {
+		return &Error{Status: 403, Msg: b.Error()}
+	}
+	msg := err.Error()
+	if strings.Contains(msg, "blocked:") {
+		// net.Dialer wraps Control errors
+		if i := strings.Index(msg, "blocked:"); i >= 0 {
+			return &Error{Status: 403, Msg: msg[i:]}
+		}
+	}
+	if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline") {
+		return errf(504, "", "upstream timeout after 20s")
+	}
+	if strings.Contains(msg, "tls:") || strings.Contains(msg, "certificate") || strings.Contains(msg, "x509") {
+		return errf(422, "browser", "upstream tls handshake failed")
+	}
+	var ue *url.Error
+	if errors.As(err, &ue) {
+		return mapDial(ue.Err)
+	}
+	return errf(422, "", "upstream error")
+}