sift/internal/fetch/fetch.go

package fetch

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"git.wntrmute.dev/kyle/sift/internal/guard"
)

const (
	DefaultMaxBody = 5 << 20
	DefaultTimeout = 20 * time.Second
)

// Resource is one successful GET.
type Resource struct {
	URL         string
	FinalURL    string
	Status      int
	ContentType string
	Body        []byte
	FetchedAt   time.Time
	Hash        string
}

func (r *Resource) SHA256() string {
	if r.Hash != "" {
		return r.Hash
	}
	sum := sha256.Sum256(r.Body)
	r.Hash = "sha256:" + hex.EncodeToString(sum[:])
	return r.Hash
}

// Error is a fetch failure with the HTTP status sift should return to its caller.
type Error struct {
	Status int
	Msg    string
	Hint   string
}

func (e *Error) Error() string { return e.Msg }

func errf(status int, hint, format string, args ...any) *Error {
	return &Error{Status: status, Hint: hint, Msg: fmt.Sprintf(format, args...)}
}

// Fetcher is the only outbound HTTP path. Guard must be the same instance
// that holds DenyPrefixes.
type Fetcher struct {
	Guard   *guard.Guard
	UA      string
	MaxBody int64
	Timeout time.Duration

	clientOnce sync.Once
	client     *http.Client
}

func (f *Fetcher) maxBody() int64 {
	if f.MaxBody <= 0 {
		return DefaultMaxBody
	}
	return f.MaxBody
}

func (f *Fetcher) timeout() time.Duration {
	if f.Timeout <= 0 {
		return DefaultTimeout
	}
	return f.Timeout
}

func (f *Fetcher) ua() string {
	if f.UA != "" {
		return f.UA
	}
	return "sift/dev"
}

func (f *Fetcher) httpClient() *http.Client {
	f.clientOnce.Do(func() {
		c := f.Guard.Client()
		c.Timeout = f.timeout()
		f.client = c
	})
	return f.client
}

// Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard.
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
	if f == nil || f.Guard == nil {
		return nil, errf(500, "", "internal error")
	}
	ctx, cancel := context.WithTimeout(ctx, f.timeout())
	defer cancel()

	if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil {
		return nil, mapGuard(err)
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
	if err != nil {
		return nil, errf(400, "", "invalid url")
	}
	req.Header.Set("User-Agent", f.ua())
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
	// no cookies, no auth, no Referer

	resp, err := f.httpClient().Do(req)
	if err != nil {
		return nil, mapDial(err)
	}
	defer resp.Body.Close()

	limit := f.maxBody()
	body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
	if err != nil {
		return nil, mapDial(err)
	}
	if int64(len(body)) > limit {
		return nil, errf(413, "", "too large (>%d MB decompressed)", limit>>20)
	}

	final := rawURL
	if resp.Request != nil && resp.Request.URL != nil {
		final = resp.Request.URL.String()
	}
	ct := resp.Header.Get("Content-Type")
	res := &Resource{
		URL:         rawURL,
		FinalURL:    final,
		Status:      resp.StatusCode,
		ContentType: ct,
		Body:        body,
		FetchedAt:   time.Now().UTC(),
	}

	switch {
	case resp.StatusCode == http.StatusOK:
		return res, nil
	case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451:
		return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode)
	case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header):
		return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser")
	case resp.StatusCode >= 400:
		return res, errf(422, "", "upstream %d", resp.StatusCode)
	default:
		return res, nil
	}
}

func looksLikeChallenge(body []byte, h http.Header) bool {
	s := strings.ToLower(string(body))
	if strings.Contains(s, "captcha") || strings.Contains(s, "cf-challenge") || strings.Contains(s, "just a moment") {
		return true
	}
	if strings.Contains(strings.ToLower(h.Get("Server")), "cloudflare") {
		return true
	}
	return false
}

func mapGuard(err error) error {
	var b *guard.BlockedError
	if errors.As(err, &b) {
		return &Error{Status: 403, Msg: b.Error()}
	}
	if strings.HasPrefix(err.Error(), "blocked:") {
		return &Error{Status: 403, Msg: err.Error()}
	}
	return &Error{Status: 403, Msg: "blocked: " + err.Error()}
}

func mapDial(err error) error {
	if err == nil {
		return nil
	}
	var b *guard.BlockedError
	if errors.As(err, &b) {
		return &Error{Status: 403, Msg: b.Error()}
	}
	msg := err.Error()
	if strings.Contains(msg, "blocked:") {
		// net.Dialer wraps Control errors
		if i := strings.Index(msg, "blocked:"); i >= 0 {
			return &Error{Status: 403, Msg: msg[i:]}
		}
	}
	if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline") {
		return errf(504, "", "upstream timeout after 20s")
	}
	if strings.Contains(msg, "tls:") || strings.Contains(msg, "certificate") || strings.Contains(msg, "x509") {
		return errf(422, "browser", "upstream tls handshake failed")
	}
	var ue *url.Error
	if errors.As(err, &ue) {
		return mapDial(ue.Err)
	}
	return errf(422, "", "upstream error")
}