clean, fetch: recognise PDFs; 100 MB / 120 s for PDFs; FetchIf revalidates

Recognise PDFs in clean.Kind (declared types and %PDF- sniffing on
octet-stream/bodyless bodies), give PDF responses the 100 MB body cap and
120 s deadline while web pages keep 5 MB / 20 s, and add FetchIf, which
sends If-None-Match / If-Modified-Since and maps a 304 to a Resource with
Status 304 and an empty Body. Fetch and FetchIf share one doFetch path for
guard, client, limits and error mapping.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

internal/clean/clean.go

@@ -28,7 +28,7 @@ type Result struct {
 	Title    string
 	Byline   string
 	Date     string
-	Kind     string // html, json, text
+	Kind     string // html, json, text, pdf
 	Warning  string
 	Trunc    bool
 }
@@ -57,6 +57,8 @@ func MediaKind(ct string) string {
 		return "json"
 	case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
 		return "text"
+	case ct == "application/pdf" || ct == "application/x-pdf":
+		return "pdf"
 	case ct == "":
 		return "sniff"
 	default:
@@ -76,6 +78,42 @@ func Sniff(body []byte) string {
 	return "text"
 }
 
+// Kind classifies a fetched body: "html", "json", "text", "pdf" or "other". The
+// media type is lower-cased and its parameters dropped, as MediaKind does.
+func Kind(contentType string, body []byte) string {
+	ct := strings.ToLower(strings.TrimSpace(contentType))
+	if i := strings.IndexByte(ct, ';'); i >= 0 {
+		ct = strings.TrimSpace(ct[:i])
+	}
+	switch {
+	case ct == "application/pdf" || ct == "application/x-pdf":
+		return "pdf"
+	case ct == "text/html" || ct == "application/xhtml+xml":
+		return "html"
+	case ct == "application/json" || strings.HasSuffix(ct, "+json"):
+		return "json"
+	case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
+		return "text"
+	case ct == "application/octet-stream" || ct == "binary/octet-stream":
+		if containsPDF(body) {
+			return "pdf"
+		}
+		return "other"
+	case ct == "":
+		if containsPDF(body) {
+			return "pdf"
+		}
+		return Sniff(body)
+	default:
+		return "other"
+	}
+}
+
+// containsPDF reports whether %PDF- appears within the first 1024 bytes.
+func containsPDF(body []byte) bool {
+	return bytes.Contains(body[:min(len(body), 1024)], []byte("%PDF-"))
+}
+
 func HTML(res *fetch.Resource, onlyMain bool, outCap int) (*Result, error) {
 	if outCap <= 0 {
 		outCap = defaultOutCap