package fetch import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "net/http" "net/url" "strings" "sync" "time" "git.wntrmute.dev/kyle/sift/internal/guard" ) const ( DefaultMaxBody = 5 << 20 DefaultTimeout = 20 * time.Second ) // Resource is one successful GET. type Resource struct { URL string FinalURL string Status int ContentType string Header http.Header Body []byte FetchedAt time.Time Hash string } func (r *Resource) SHA256() string { if r.Hash != "" { return r.Hash } sum := sha256.Sum256(r.Body) r.Hash = "sha256:" + hex.EncodeToString(sum[:]) return r.Hash } // Error is a fetch failure with the HTTP status sift should return to its caller. type Error struct { Status int Msg string Hint string RetryAfter int // seconds to wait before retrying; 0 = none } func (e *Error) Error() string { return e.Msg } func errf(status int, hint, format string, args ...any) *Error { return &Error{Status: status, Hint: hint, Msg: fmt.Sprintf(format, args...)} } // Fetcher is the only outbound HTTP path. Guard must be the same instance // that holds DenyPrefixes. type Fetcher struct { Guard *guard.Guard UA string MaxBody int64 Timeout time.Duration // PDFMaxBody and PDFTimeout apply to a PDF response; 0 means the // DefaultPDF* values. They live here (not in pdf.go) because they are // fields on the Fetcher, whose base definition stays in fetch.go. PDFMaxBody int64 // 0 = DefaultPDFMaxBody PDFTimeout time.Duration // 0 = DefaultPDFTimeout clientOnce sync.Once client *http.Client } func (f *Fetcher) maxBody() int64 { if f.MaxBody <= 0 { return DefaultMaxBody } return f.MaxBody } func (f *Fetcher) timeout() time.Duration { if f.Timeout <= 0 { return DefaultTimeout } return f.Timeout } func (f *Fetcher) ua() string { if f.UA != "" { return f.UA } return "sift/dev" } func (f *Fetcher) httpClient() *http.Client { f.clientOnce.Do(func() { c := f.Guard.Client() c.Timeout = f.clientTimeout() f.client = c }) return f.client } // Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard. func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) { return f.doFetch(ctx, rawURL, "", "") } func looksLikeChallenge(body []byte, h http.Header) bool { s := strings.ToLower(string(body)) if strings.Contains(s, "captcha") || strings.Contains(s, "cf-challenge") || strings.Contains(s, "just a moment") { return true } if strings.Contains(strings.ToLower(h.Get("Server")), "cloudflare") { return true } return false } func mapGuard(err error) error { var b *guard.BlockedError if errors.As(err, &b) { return &Error{Status: 403, Msg: b.Error()} } if strings.HasPrefix(err.Error(), "blocked:") { return &Error{Status: 403, Msg: err.Error()} } return &Error{Status: 403, Msg: "blocked: " + err.Error()} } func mapDial(err error) error { if err == nil { return nil } var b *guard.BlockedError if errors.As(err, &b) { return &Error{Status: 403, Msg: b.Error()} } msg := err.Error() if strings.Contains(msg, "blocked:") { // net.Dialer wraps Control errors if i := strings.Index(msg, "blocked:"); i >= 0 { return &Error{Status: 403, Msg: msg[i:]} } } if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline") { return errf(504, "", "upstream timeout after 20s") } if strings.Contains(msg, "tls:") || strings.Contains(msg, "certificate") || strings.Contains(msg, "x509") { return errf(422, "browser", "upstream tls handshake failed") } var ue *url.Error if errors.As(err, &ue) { return mapDial(ue.Err) } return errf(422, "", "upstream error") }