clean, fetch: recognise PDFs; 100 MB / 120 s for PDFs; FetchIf revalidates

Recognise PDFs in clean.Kind (declared types and %PDF- sniffing on
octet-stream/bodyless bodies), give PDF responses the 100 MB body cap and
120 s deadline while web pages keep 5 MB / 20 s, and add FetchIf, which
sends If-None-Match / If-Modified-Since and maps a 304 to a Resource with
Status 304 and an empty Body. Fetch and FetchIf share one doFetch path for
guard, client, limits and error mapping.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

internal/clean/clean.go

@@ -28,7 +28,7 @@ type Result struct {
 	Title    string
 	Byline   string
 	Date     string
-	Kind     string // html, json, text
+	Kind     string // html, json, text, pdf
 	Warning  string
 	Trunc    bool
 }
@@ -57,6 +57,8 @@ func MediaKind(ct string) string {
 		return "json"
 	case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
 		return "text"
+	case ct == "application/pdf" || ct == "application/x-pdf":
+		return "pdf"
 	case ct == "":
 		return "sniff"
 	default:
@@ -76,6 +78,42 @@ func Sniff(body []byte) string {
 	return "text"
 }
 
+// Kind classifies a fetched body: "html", "json", "text", "pdf" or "other". The
+// media type is lower-cased and its parameters dropped, as MediaKind does.
+func Kind(contentType string, body []byte) string {
+	ct := strings.ToLower(strings.TrimSpace(contentType))
+	if i := strings.IndexByte(ct, ';'); i >= 0 {
+		ct = strings.TrimSpace(ct[:i])
+	}
+	switch {
+	case ct == "application/pdf" || ct == "application/x-pdf":
+		return "pdf"
+	case ct == "text/html" || ct == "application/xhtml+xml":
+		return "html"
+	case ct == "application/json" || strings.HasSuffix(ct, "+json"):
+		return "json"
+	case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
+		return "text"
+	case ct == "application/octet-stream" || ct == "binary/octet-stream":
+		if containsPDF(body) {
+			return "pdf"
+		}
+		return "other"
+	case ct == "":
+		if containsPDF(body) {
+			return "pdf"
+		}
+		return Sniff(body)
+	default:
+		return "other"
+	}
+}
+
+// containsPDF reports whether %PDF- appears within the first 1024 bytes.
+func containsPDF(body []byte) bool {
+	return bytes.Contains(body[:min(len(body), 1024)], []byte("%PDF-"))
+}
+
 func HTML(res *fetch.Resource, onlyMain bool, outCap int) (*Result, error) {
 	if outCap <= 0 {
 		outCap = defaultOutCap

internal/clean/kind_test.go

@@ -0,0 +1,55 @@
+package clean_test
+
+// P5a task 01: Kind decides which cleaner a fetched body gets, now including "pdf".
+
+import (
+	"strings"
+	"testing"
+
+	"git.wntrmute.dev/kyle/sift/internal/clean"
+)
+
+func TestKind(t *testing.T) {
+	pdf := []byte("%PDF-1.7\n%\xe2\xe3\xcf\xd3\n1 0 obj << >> endobj\n")
+	junkThenPDF := append([]byte(strings.Repeat(" ", 900)), pdf...) // the spec allows leading junk
+	lateMagic := append([]byte(strings.Repeat("x", 2000)), pdf...)  // past the first 1024 bytes
+	binary := []byte{0x50, 0x4b, 0x03, 0x04, 0x00, 0x00}            // a zip
+	for _, tc := range []struct {
+		name, ct string
+		body     []byte
+		want     string
+	}{
+		{"pdf by type", "application/pdf", pdf, "pdf"},
+		{"pdf by type with params", "Application/PDF; qs=0.9", pdf, "pdf"},
+		{"x-pdf", "application/x-pdf", pdf, "pdf"},
+		{"pdf type, body not checked", "application/pdf", []byte("oops"), "pdf"},
+		{"octet-stream sniffed as pdf", "application/octet-stream", pdf, "pdf"},
+		{"binary/octet-stream sniffed as pdf", "binary/octet-stream", pdf, "pdf"},
+		{"no type sniffed as pdf", "", pdf, "pdf"},
+		{"leading junk within 1024 bytes", "application/octet-stream", junkThenPDF, "pdf"},
+		{"magic too late", "application/octet-stream", lateMagic, "other"},
+		{"octet-stream that is not a pdf", "application/octet-stream", binary, "other"},
+		{"html stays html even if it mentions %PDF-", "text/html; charset=utf-8", []byte("<html>%PDF-1.4</html>"), "html"},
+		{"json", "application/json", []byte(`{"a":1}`), "json"},
+		{"problem+json", "application/problem+json", []byte(`{}`), "json"},
+		{"text", "text/plain", []byte("hello"), "text"},
+		{"no type, html", "", []byte("<!doctype html><html></html>"), "html"},
+		{"no type, json", "", []byte(`[1,2]`), "json"},
+		{"no type, text", "", []byte("just words"), "text"},
+		{"zip", "application/zip", binary, "other"},
+		{"image", "image/png", []byte{0x89, 'P', 'N', 'G'}, "other"},
+	} {
+		if got := clean.Kind(tc.ct, tc.body); got != tc.want {
+			t.Errorf("%s: Kind(%q, …) = %q, want %q", tc.name, tc.ct, got, tc.want)
+		}
+	}
+}
+
+// MediaKind keeps answering for callers that have no body: the pdf types are "pdf" there too.
+func TestMediaKindKnowsPDF(t *testing.T) {
+	for ct, want := range map[string]string{"application/pdf": "pdf", "application/x-pdf": "pdf", "text/html": "html", "": "sniff"} {
+		if got := clean.MediaKind(ct); got != want {
+			t.Errorf("MediaKind(%q) = %q, want %q", ct, got, want)
+		}
+	}
+}