clean, fetch: recognise PDFs; 100 MB / 120 s for PDFs; FetchIf revalidates

Recognise PDFs in clean.Kind (declared types and %PDF- sniffing on
octet-stream/bodyless bodies), give PDF responses the 100 MB body cap and
120 s deadline while web pages keep 5 MB / 20 s, and add FetchIf, which
sends If-None-Match / If-Modified-Since and maps a 304 to a Resource with
Status 304 and an empty Body. Fetch and FetchIf share one doFetch path for
guard, client, limits and error mapping.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

internal/fetch/fetch.go

@@ -6,7 +6,6 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -62,6 +61,11 @@ type Fetcher struct {
 	UA      string
 	MaxBody int64
 	Timeout time.Duration
+	// PDFMaxBody and PDFTimeout apply to a PDF response; 0 means the
+	// DefaultPDF* values. They live here (not in pdf.go) because they are
+	// fields on the Fetcher, whose base definition stays in fetch.go.
+	PDFMaxBody int64         // 0 = DefaultPDFMaxBody
+	PDFTimeout time.Duration // 0 = DefaultPDFTimeout
 
 	clientOnce sync.Once
 	client     *http.Client
@@ -91,7 +95,7 @@ func (f *Fetcher) ua() string {
 func (f *Fetcher) httpClient() *http.Client {
 	f.clientOnce.Do(func() {
 		c := f.Guard.Client()
-		c.Timeout = f.timeout()
+		c.Timeout = f.clientTimeout()
 		f.client = c
 	})
 	return f.client
@@ -99,66 +103,7 @@ func (f *Fetcher) httpClient() *http.Client {
 
 // Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard.
 func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
-	if f == nil || f.Guard == nil {
-		return nil, errf(500, "", "internal error")
-	}
-	ctx, cancel := context.WithTimeout(ctx, f.timeout())
-	defer cancel()
-
-	if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil {
-		return nil, mapGuard(err)
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
-	if err != nil {
-		return nil, errf(400, "", "invalid url")
-	}
-	req.Header.Set("User-Agent", f.ua())
-	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
-	// no cookies, no auth, no Referer
-
-	resp, err := f.httpClient().Do(req)
-	if err != nil {
-		return nil, mapDial(err)
-	}
-	defer resp.Body.Close()
-
-	limit := f.maxBody()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
-	if err != nil {
-		return nil, mapDial(err)
-	}
-	if int64(len(body)) > limit {
-		return nil, errf(413, "", "too large (>%d MB decompressed)", limit>>20)
-	}
-
-	final := rawURL
-	if resp.Request != nil && resp.Request.URL != nil {
-		final = resp.Request.URL.String()
-	}
-	ct := resp.Header.Get("Content-Type")
-	res := &Resource{
-		URL:         rawURL,
-		FinalURL:    final,
-		Status:      resp.StatusCode,
-		ContentType: ct,
-		Header:      resp.Header.Clone(),
-		Body:        body,
-		FetchedAt:   time.Now().UTC(),
-	}
-
-	switch {
-	case resp.StatusCode == http.StatusOK:
-		return res, nil
-	case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451:
-		return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode)
-	case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header):
-		return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser")
-	case resp.StatusCode >= 400:
-		return res, errf(422, "", "upstream %d", resp.StatusCode)
-	default:
-		return res, nil
-	}
+	return f.doFetch(ctx, rawURL, "", "")
 }
 
 func looksLikeChallenge(body []byte, h http.Header) bool {