sift/internal/fetch/fetch.go

package fetch

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"git.wntrmute.dev/kyle/sift/internal/guard"
)

const (
	DefaultMaxBody = 5 << 20
	DefaultTimeout = 20 * time.Second
)

// Resource is one successful GET.
type Resource struct {
	URL         string
	FinalURL    string
	Status      int
	ContentType string
	Header      http.Header
	Body        []byte
	FetchedAt   time.Time
	Hash        string
}

func (r *Resource) SHA256() string {
	if r.Hash != "" {
		return r.Hash
	}
	sum := sha256.Sum256(r.Body)
	r.Hash = "sha256:" + hex.EncodeToString(sum[:])
	return r.Hash
}

// Error is a fetch failure with the HTTP status sift should return to its caller.
type Error struct {
	Status     int
	Msg        string
	Hint       string
	RetryAfter int // seconds to wait before retrying; 0 = none
}

func (e *Error) Error() string { return e.Msg }

func errf(status int, hint, format string, args ...any) *Error {
	return &Error{Status: status, Hint: hint, Msg: fmt.Sprintf(format, args...)}
}

// Fetcher is the only outbound HTTP path. Guard must be the same instance
// that holds DenyPrefixes.
type Fetcher struct {
	Guard   *guard.Guard
	UA      string
	MaxBody int64
	Timeout time.Duration
	// PDFMaxBody and PDFTimeout apply to a PDF response; 0 means the
	// DefaultPDF* values. They live here (not in pdf.go) because they are
	// fields on the Fetcher, whose base definition stays in fetch.go.
	PDFMaxBody int64         // 0 = DefaultPDFMaxBody
	PDFTimeout time.Duration // 0 = DefaultPDFTimeout

	clientOnce sync.Once
	client     *http.Client
}

func (f *Fetcher) maxBody() int64 {
	if f.MaxBody <= 0 {
		return DefaultMaxBody
	}
	return f.MaxBody
}

func (f *Fetcher) timeout() time.Duration {
	if f.Timeout <= 0 {
		return DefaultTimeout
	}
	return f.Timeout
}

func (f *Fetcher) ua() string {
	if f.UA != "" {
		return f.UA
	}
	return "sift/dev"
}

func (f *Fetcher) httpClient() *http.Client {
	f.clientOnce.Do(func() {
		c := f.Guard.Client()
		c.Timeout = f.clientTimeout()
		f.client = c
	})
	return f.client
}

// Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard.
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
	return f.doFetch(ctx, rawURL, "", "")
}

func looksLikeChallenge(body []byte, h http.Header) bool {
	s := strings.ToLower(string(body))
	if strings.Contains(s, "captcha") || strings.Contains(s, "cf-challenge") || strings.Contains(s, "just a moment") {
		return true
	}
	if strings.Contains(strings.ToLower(h.Get("Server")), "cloudflare") {
		return true
	}
	return false
}

func mapGuard(err error) error {
	var b *guard.BlockedError
	if errors.As(err, &b) {
		return &Error{Status: 403, Msg: b.Error()}
	}
	if strings.HasPrefix(err.Error(), "blocked:") {
		return &Error{Status: 403, Msg: err.Error()}
	}
	return &Error{Status: 403, Msg: "blocked: " + err.Error()}
}

func mapDial(err error) error {
	if err == nil {
		return nil
	}
	var b *guard.BlockedError
	if errors.As(err, &b) {
		return &Error{Status: 403, Msg: b.Error()}
	}
	msg := err.Error()
	if strings.Contains(msg, "blocked:") {
		// net.Dialer wraps Control errors
		if i := strings.Index(msg, "blocked:"); i >= 0 {
			return &Error{Status: 403, Msg: msg[i:]}
		}
	}
	if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline") {
		return errf(504, "", "upstream timeout after 20s")
	}
	if strings.Contains(msg, "tls:") || strings.Contains(msg, "certificate") || strings.Contains(msg, "x509") {
		return errf(422, "browser", "upstream tls handshake failed")
	}
	var ue *url.Error
	if errors.As(err, &ue) {
		return mapDial(ue.Err)
	}
	return errf(422, "", "upstream error")
}