package fetch import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "net/http" "net/url" "strings" "time" "git.wntrmute.dev/kyle/sift/internal/guard" ) const ( DefaultMaxBody = 5 << 20 DefaultTimeout = 20 * time.Second ) // Resource is one successful GET. type Resource struct { URL string FinalURL string Status int ContentType string Body []byte FetchedAt time.Time Hash string } func (r *Resource) SHA256() string { if r.Hash != "" { return r.Hash } sum := sha256.Sum256(r.Body) r.Hash = "sha256:" + hex.EncodeToString(sum[:]) return r.Hash } // Error is a fetch failure with the HTTP status sift should return to its caller. type Error struct { Status int Msg string Hint string } func (e *Error) Error() string { return e.Msg } func errf(status int, hint, format string, args ...any) *Error { return &Error{Status: status, Hint: hint, Msg: fmt.Sprintf(format, args...)} } // Fetcher is the only outbound HTTP path. Guard must be the same instance // that holds DenyPrefixes. type Fetcher struct { Guard *guard.Guard UA string MaxBody int64 Timeout time.Duration } func (f *Fetcher) maxBody() int64 { if f.MaxBody <= 0 { return DefaultMaxBody } return f.MaxBody } func (f *Fetcher) timeout() time.Duration { if f.Timeout <= 0 { return DefaultTimeout } return f.Timeout } func (f *Fetcher) ua() string { if f.UA != "" { return f.UA } return "sift/dev" } // Fetch GETs rawURL after Guard.CheckURL, using Guard.Client() from the same instance. func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) { if f == nil || f.Guard == nil { return nil, errf(500, "", "internal error") } ctx, cancel := context.WithTimeout(ctx, f.timeout()) defer cancel() if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil { return nil, mapGuard(err) } req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return nil, errf(400, "", "invalid url") } req.Header.Set("User-Agent", f.ua()) req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1") // no cookies, no auth, no Referer client := f.Guard.Client() client.Timeout = f.timeout() resp, err := client.Do(req) if err != nil { return nil, mapDial(err) } defer resp.Body.Close() limit := f.maxBody() body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) if err != nil { return nil, mapDial(err) } if int64(len(body)) > limit { return nil, errf(413, "", "too large (>%d MB decompressed)", limit>>20) } final := rawURL if resp.Request != nil && resp.Request.URL != nil { final = resp.Request.URL.String() } ct := resp.Header.Get("Content-Type") if i := strings.IndexByte(ct, ';'); i >= 0 { // keep full header for charset; also store raw } res := &Resource{ URL: rawURL, FinalURL: final, Status: resp.StatusCode, ContentType: ct, Body: body, FetchedAt: time.Now().UTC(), } switch { case resp.StatusCode == http.StatusOK: return res, nil case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451: return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode) case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header): return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser") case resp.StatusCode >= 400: return res, errf(422, "", "upstream %d", resp.StatusCode) default: return res, nil } } func looksLikeChallenge(body []byte, h http.Header) bool { s := strings.ToLower(string(body)) if strings.Contains(s, "captcha") || strings.Contains(s, "cf-challenge") || strings.Contains(s, "just a moment") { return true } if strings.Contains(strings.ToLower(h.Get("Server")), "cloudflare") { return true } return false } func mapGuard(err error) error { var b *guard.BlockedError if errors.As(err, &b) { return &Error{Status: 403, Msg: b.Error()} } if strings.HasPrefix(err.Error(), "blocked:") { return &Error{Status: 403, Msg: err.Error()} } return &Error{Status: 403, Msg: "blocked: " + err.Error()} } func mapDial(err error) error { if err == nil { return nil } var b *guard.BlockedError if errors.As(err, &b) { return &Error{Status: 403, Msg: b.Error()} } msg := err.Error() if strings.Contains(msg, "blocked:") { // net.Dialer wraps Control errors if i := strings.Index(msg, "blocked:"); i >= 0 { return &Error{Status: 403, Msg: msg[i:]} } } if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline") { return errf(504, "", "upstream timeout after 20s") } if strings.Contains(msg, "tls:") || strings.Contains(msg, "certificate") || strings.Contains(msg, "x509") { return errf(422, "browser", "upstream tls handshake failed") } var ue *url.Error if errors.As(err, &ue) { return mapDial(ue.Err) } return errf(422, "", "upstream error") }