clean, fetch: recognise PDFs; 100 MB / 120 s for PDFs; FetchIf revalidates

Recognise PDFs in clean.Kind (declared types and %PDF- sniffing on
octet-stream/bodyless bodies), give PDF responses the 100 MB body cap and
120 s deadline while web pages keep 5 MB / 20 s, and add FetchIf, which
sends If-None-Match / If-Modified-Since and maps a 304 to a Resource with
Status 304 and an empty Body. Fetch and FetchIf share one doFetch path for
guard, client, limits and error mapping.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
2026-09-25 22:40:04 -07:00
co-authored by ornith-1.5-35b-a3b
parent 6a3b6dab71
commit 13365a2daa
6 changed files with 514 additions and 63 deletions
+1
View File
@@ -4,3 +4,4 @@ One row per task session that ends in a commit. Newest last.
| Task | Date | Status | Sessions | Gate | Deviations | Summary |
|---|---|---|---|---|---|---|
| p5a/01-kind-fetch | 2026-09-25 | done | resumed | gate: ok | none | Implemented Kind/`application/pdf` recognition, PDF body cap (100 MB) and deadline (120 s), and FetchIf with If-None-Match/If-Modified-Since via shared doFetch. Previous session could not pass TestFetchIf only because the given test held its mutex across the handler (deadlock); owner fixed the test. Session resumed after that fix — did not rewrite fetch code, only verified rules and ran gate. |
+39 -1
View File
@@ -28,7 +28,7 @@ type Result struct {
Title string
Byline string
Date string
Kind string // html, json, text
Kind string // html, json, text, pdf
Warning string
Trunc bool
}
@@ -57,6 +57,8 @@ func MediaKind(ct string) string {
return "json"
case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
return "text"
case ct == "application/pdf" || ct == "application/x-pdf":
return "pdf"
case ct == "":
return "sniff"
default:
@@ -76,6 +78,42 @@ func Sniff(body []byte) string {
return "text"
}
// Kind classifies a fetched body: "html", "json", "text", "pdf" or "other". The
// media type is lower-cased and its parameters dropped, as MediaKind does.
func Kind(contentType string, body []byte) string {
ct := strings.ToLower(strings.TrimSpace(contentType))
if i := strings.IndexByte(ct, ';'); i >= 0 {
ct = strings.TrimSpace(ct[:i])
}
switch {
case ct == "application/pdf" || ct == "application/x-pdf":
return "pdf"
case ct == "text/html" || ct == "application/xhtml+xml":
return "html"
case ct == "application/json" || strings.HasSuffix(ct, "+json"):
return "json"
case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
return "text"
case ct == "application/octet-stream" || ct == "binary/octet-stream":
if containsPDF(body) {
return "pdf"
}
return "other"
case ct == "":
if containsPDF(body) {
return "pdf"
}
return Sniff(body)
default:
return "other"
}
}
// containsPDF reports whether %PDF- appears within the first 1024 bytes.
func containsPDF(body []byte) bool {
return bytes.Contains(body[:min(len(body), 1024)], []byte("%PDF-"))
}
func HTML(res *fetch.Resource, onlyMain bool, outCap int) (*Result, error) {
if outCap <= 0 {
outCap = defaultOutCap
+55
View File
@@ -0,0 +1,55 @@
package clean_test
// P5a task 01: Kind decides which cleaner a fetched body gets, now including "pdf".
import (
"strings"
"testing"
"git.wntrmute.dev/kyle/sift/internal/clean"
)
func TestKind(t *testing.T) {
pdf := []byte("%PDF-1.7\n%\xe2\xe3\xcf\xd3\n1 0 obj << >> endobj\n")
junkThenPDF := append([]byte(strings.Repeat(" ", 900)), pdf...) // the spec allows leading junk
lateMagic := append([]byte(strings.Repeat("x", 2000)), pdf...) // past the first 1024 bytes
binary := []byte{0x50, 0x4b, 0x03, 0x04, 0x00, 0x00} // a zip
for _, tc := range []struct {
name, ct string
body []byte
want string
}{
{"pdf by type", "application/pdf", pdf, "pdf"},
{"pdf by type with params", "Application/PDF; qs=0.9", pdf, "pdf"},
{"x-pdf", "application/x-pdf", pdf, "pdf"},
{"pdf type, body not checked", "application/pdf", []byte("oops"), "pdf"},
{"octet-stream sniffed as pdf", "application/octet-stream", pdf, "pdf"},
{"binary/octet-stream sniffed as pdf", "binary/octet-stream", pdf, "pdf"},
{"no type sniffed as pdf", "", pdf, "pdf"},
{"leading junk within 1024 bytes", "application/octet-stream", junkThenPDF, "pdf"},
{"magic too late", "application/octet-stream", lateMagic, "other"},
{"octet-stream that is not a pdf", "application/octet-stream", binary, "other"},
{"html stays html even if it mentions %PDF-", "text/html; charset=utf-8", []byte("<html>%PDF-1.4</html>"), "html"},
{"json", "application/json", []byte(`{"a":1}`), "json"},
{"problem+json", "application/problem+json", []byte(`{}`), "json"},
{"text", "text/plain", []byte("hello"), "text"},
{"no type, html", "", []byte("<!doctype html><html></html>"), "html"},
{"no type, json", "", []byte(`[1,2]`), "json"},
{"no type, text", "", []byte("just words"), "text"},
{"zip", "application/zip", binary, "other"},
{"image", "image/png", []byte{0x89, 'P', 'N', 'G'}, "other"},
} {
if got := clean.Kind(tc.ct, tc.body); got != tc.want {
t.Errorf("%s: Kind(%q, …) = %q, want %q", tc.name, tc.ct, got, tc.want)
}
}
}
// MediaKind keeps answering for callers that have no body: the pdf types are "pdf" there too.
func TestMediaKindKnowsPDF(t *testing.T) {
for ct, want := range map[string]string{"application/pdf": "pdf", "application/x-pdf": "pdf", "text/html": "html", "": "sniff"} {
if got := clean.MediaKind(ct); got != want {
t.Errorf("MediaKind(%q) = %q, want %q", ct, got, want)
}
}
}
+7 -62
View File
@@ -6,7 +6,6 @@ import (
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
@@ -62,6 +61,11 @@ type Fetcher struct {
UA string
MaxBody int64
Timeout time.Duration
// PDFMaxBody and PDFTimeout apply to a PDF response; 0 means the
// DefaultPDF* values. They live here (not in pdf.go) because they are
// fields on the Fetcher, whose base definition stays in fetch.go.
PDFMaxBody int64 // 0 = DefaultPDFMaxBody
PDFTimeout time.Duration // 0 = DefaultPDFTimeout
clientOnce sync.Once
client *http.Client
@@ -91,7 +95,7 @@ func (f *Fetcher) ua() string {
func (f *Fetcher) httpClient() *http.Client {
f.clientOnce.Do(func() {
c := f.Guard.Client()
c.Timeout = f.timeout()
c.Timeout = f.clientTimeout()
f.client = c
})
return f.client
@@ -99,66 +103,7 @@ func (f *Fetcher) httpClient() *http.Client {
// Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard.
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
if f == nil || f.Guard == nil {
return nil, errf(500, "", "internal error")
}
ctx, cancel := context.WithTimeout(ctx, f.timeout())
defer cancel()
if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil {
return nil, mapGuard(err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, errf(400, "", "invalid url")
}
req.Header.Set("User-Agent", f.ua())
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
// no cookies, no auth, no Referer
resp, err := f.httpClient().Do(req)
if err != nil {
return nil, mapDial(err)
}
defer resp.Body.Close()
limit := f.maxBody()
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err != nil {
return nil, mapDial(err)
}
if int64(len(body)) > limit {
return nil, errf(413, "", "too large (>%d MB decompressed)", limit>>20)
}
final := rawURL
if resp.Request != nil && resp.Request.URL != nil {
final = resp.Request.URL.String()
}
ct := resp.Header.Get("Content-Type")
res := &Resource{
URL: rawURL,
FinalURL: final,
Status: resp.StatusCode,
ContentType: ct,
Header: resp.Header.Clone(),
Body: body,
FetchedAt: time.Now().UTC(),
}
switch {
case resp.StatusCode == http.StatusOK:
return res, nil
case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451:
return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode)
case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header):
return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser")
case resp.StatusCode >= 400:
return res, errf(422, "", "upstream %d", resp.StatusCode)
default:
return res, nil
}
return f.doFetch(ctx, rawURL, "", "")
}
func looksLikeChallenge(body []byte, h http.Header) bool {
+233
View File
@@ -0,0 +1,233 @@
package fetch
import (
"bytes"
"context"
"io"
"net/http"
"strings"
"time"
)
const (
// DefaultPDFMaxBody is the body cap for a PDF response: 100 MiB, versus the
// 5 MiB web cap (DefaultMaxBody).
DefaultPDFMaxBody = 100 << 20
// DefaultPDFTimeout is the deadline for a PDF response: 120 s, versus the
// 20 s web deadline (DefaultTimeout).
DefaultPDFTimeout = 120 * time.Second
)
// magicPDF marks the start of a PDF, found within the first sniffWindow bytes of
// an octet-stream or a bodyless response.
var magicPDF = []byte("%PDF-")
// sniffWindow is how many leading bytes are inspected to recognise a PDF whose
// Content-Type does not say so.
const sniffWindow = 1024
func (f *Fetcher) pdfMaxBody() int64 {
if f.PDFMaxBody <= 0 {
return DefaultPDFMaxBody
}
return f.PDFMaxBody
}
func (f *Fetcher) pdfTimeout() time.Duration {
if f.PDFTimeout <= 0 {
return DefaultPDFTimeout
}
return f.PDFTimeout
}
// clientTimeout is the http.Client.Timeout: it must cover the longest deadline
// this Fetcher can give a request, so a PDF is not cut off at the web deadline.
func (f *Fetcher) clientTimeout() time.Duration {
t := f.timeout()
if p := f.pdfTimeout(); p > t {
return p
}
return t
}
// normalizeContentType lower-cases a media type and drops its parameters.
func normalizeContentType(ct string) string {
ct = strings.ToLower(strings.TrimSpace(ct))
if i := strings.IndexByte(ct, ';'); i >= 0 {
ct = strings.TrimSpace(ct[:i])
}
return ct
}
// isPDFContentType reports whether a media type is unambiguously a PDF.
func isPDFContentType(ct string) bool {
ct = normalizeContentType(ct)
return ct == "application/pdf" || ct == "application/x-pdf"
}
// FetchIf GETs rawURL like Fetch but adds conditional-GET validators: a non-empty
// etag is sent as If-None-Match and a non-empty lastModified as If-Modified-Since.
// A 304 answer returns a Resource with Status 304, an empty Body and a nil error.
// With both empty it is exactly Fetch.
func (f *Fetcher) FetchIf(ctx context.Context, rawURL, etag, lastModified string) (*Resource, error) {
return f.doFetch(ctx, rawURL, etag, lastModified)
}
// doFetch is the shared body of Fetch and FetchIf: one guard check, one client,
// one set of limits and error mappings, differing only in request headers.
func (f *Fetcher) doFetch(ctx context.Context, rawURL, etag, lastModified string) (*Resource, error) {
if f == nil || f.Guard == nil {
return nil, errf(500, "", "internal error")
}
// A PDF may take longer than a web page. Give the request the PDF deadline
// and cancel it at the ordinary (web) deadline unless the response is
// recognised as a PDF sooner, by headers or by the first bytes sniffed.
pdfDeadline := f.pdfTimeout()
ctx, cancel := context.WithTimeout(ctx, pdfDeadline)
var webTimer *time.Timer
if web := f.timeout(); web < pdfDeadline {
webTimer = time.AfterFunc(web, cancel)
}
if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil {
cancel()
return nil, mapGuard(err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
cancel()
return nil, errf(400, "", "invalid url")
}
req.Header.Set("User-Agent", f.ua())
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
if etag != "" {
req.Header.Set("If-None-Match", etag)
}
if lastModified != "" {
req.Header.Set("If-Modified-Since", lastModified)
}
// no cookies, no auth, no Referer
resp, err := f.httpClient().Do(req)
if err != nil {
cancel()
return nil, mapDial(err)
}
defer resp.Body.Close()
// The headers are in; a declared PDF type no longer needs the web deadline.
ct := resp.Header.Get("Content-Type")
pdfByHeader := isPDFContentType(ct)
if pdfByHeader && webTimer != nil {
webTimer.Stop()
}
body, err := f.readBody(resp, ct, pdfByHeader, func() {
// Sniffing an octet-stream or bodyless response as a PDF: keep the PDF
// deadline instead of cancelling at the web deadline.
if webTimer != nil {
webTimer.Stop()
}
})
if err != nil {
cancel()
return nil, err
}
final := rawURL
if resp.Request != nil && resp.Request.URL != nil {
final = resp.Request.URL.String()
}
res := &Resource{
URL: rawURL,
FinalURL: final,
Status: resp.StatusCode,
ContentType: ct,
Header: resp.Header.Clone(),
Body: body,
FetchedAt: time.Now().UTC(),
}
switch {
case resp.StatusCode == http.StatusOK:
cancel()
return res, nil
case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451:
cancel()
return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode)
case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header):
cancel()
return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser")
case resp.StatusCode == http.StatusNotModified:
cancel()
_, _ = io.Copy(io.Discard, resp.Body)
return &Resource{
URL: rawURL,
FinalURL: final,
Status: resp.StatusCode,
ContentType: ct,
Header: resp.Header.Clone(),
}, nil
case resp.StatusCode >= 400:
cancel()
return res, errf(422, "", "upstream %d", resp.StatusCode)
default:
cancel()
return res, nil
}
}
// readBody reads resp.Body under the cap that applies and reports whether the
// response is a PDF (by declared type or by sniffing an octet-stream/bodyless
// body). onPDF, when non-nil, is called once the body is sniffed as a PDF so the
// caller can drop the web deadline.
func (f *Fetcher) readBody(resp *http.Response, ct string, pdfByHeader bool, onPDF func()) ([]byte, error) {
norm := normalizeContentType(ct)
if !pdfByHeader && norm != "application/octet-stream" && norm != "binary/octet-stream" && norm != "" {
// A known non-PDF type: the web cap, no sniffing.
body, err := io.ReadAll(io.LimitReader(resp.Body, f.maxBody()+1))
if err != nil {
return nil, mapDial(err)
}
if int64(len(body)) > f.maxBody() {
return nil, errf(413, "", "too large (>%d MB decompressed)", f.maxBody()>>20)
}
return body, nil
}
// PDF by declared type, or a type that may hide one: sniff the leading bytes
// to pick the cap, then read under it.
pdf := pdfByHeader
cap := f.maxBody()
if pdf {
cap = f.pdfMaxBody()
}
var body []byte
if !pdfByHeader {
head, err := io.ReadAll(io.LimitReader(resp.Body, sniffWindow))
if err != nil {
return nil, mapDial(err)
}
if bytes.Contains(head, magicPDF) {
pdf = true
cap = f.pdfMaxBody()
if onPDF != nil {
onPDF()
}
}
body = head
}
if remaining := cap + 1 - int64(len(body)); remaining > 0 {
more, err := io.ReadAll(io.LimitReader(resp.Body, remaining))
if err != nil {
return nil, mapDial(err)
}
body = append(body, more...)
}
if int64(len(body)) > cap {
return nil, errf(413, "", "too large (>%d MB decompressed)", cap>>20)
}
return body, nil
}
+179
View File
@@ -0,0 +1,179 @@
package fetch_test
// P5a task 01: a PDF response gets a larger body cap and a longer deadline than a web page, and
// FetchIf revalidates a cached document with a conditional GET.
import (
"bytes"
"context"
"errors"
"net/http"
"net/http/httptest"
"net/netip"
"strings"
"sync"
"testing"
"time"
"git.wntrmute.dev/kyle/sift/internal/fetch"
"git.wntrmute.dev/kyle/sift/internal/guard"
)
func loopbackFetcher() *fetch.Fetcher {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
return &fetch.Fetcher{Guard: g, UA: "sift-test"}
}
func body(prefix string, n int) []byte {
b := bytes.Repeat([]byte("x"), n)
copy(b, prefix)
return b
}
func serve(t *testing.T, ct string, b []byte) string {
t.Helper()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if ct != "" {
w.Header().Set("Content-Type", ct)
}
_, _ = w.Write(b)
}))
t.Cleanup(ts.Close)
return ts.URL
}
func status(err error) int {
var fe *fetch.Error
if errors.As(err, &fe) {
return fe.Status
}
return 0
}
func TestPDFBodyCap(t *testing.T) {
six := 6 << 20 // over DefaultMaxBody (5 MiB), under DefaultPDFMaxBody (100 MiB)
for _, tc := range []struct {
name, ct string
b []byte
want int // 0 = ok
}{
{"pdf by type", "application/pdf", body("%PDF-1.7\n", six), 0},
{"pdf by sniffing octet-stream", "application/octet-stream", body("%PDF-1.7\n", six), 0},
{"pdf with no type", "", body("%PDF-1.7\n", six), 0},
{"html keeps the 5 MiB cap", "text/html", body("<html>", six), 413},
{"octet-stream that is not a pdf keeps the cap", "application/octet-stream", body("PK\x03\x04", six), 413},
} {
f := loopbackFetcher()
res, err := f.Fetch(context.Background(), serve(t, tc.ct, tc.b))
if tc.want == 0 {
if err != nil || res == nil || len(res.Body) != six {
t.Errorf("%s: err %v, body %d bytes; want the whole %d", tc.name, err, lenBody(res), six)
}
continue
}
if status(err) != tc.want {
t.Errorf("%s: err %v (status %d), want %d", tc.name, err, status(err), tc.want)
}
}
}
func lenBody(r *fetch.Resource) int {
if r == nil {
return -1
}
return len(r.Body)
}
func TestPDFMaxBodyIsConfigurable(t *testing.T) {
f := loopbackFetcher()
f.PDFMaxBody = 1 << 20
_, err := f.Fetch(context.Background(), serve(t, "application/pdf", body("%PDF-1.7\n", 2<<20)))
if status(err) != 413 || !strings.Contains(err.Error(), "too large") {
t.Errorf("2 MiB pdf over a 1 MiB PDFMaxBody: %v, want 413 too large", err)
}
if fetch.DefaultPDFMaxBody != 100<<20 || fetch.DefaultPDFTimeout != 120*time.Second {
t.Errorf("defaults = %d, %v; want 100 MiB, 120s", fetch.DefaultPDFMaxBody, fetch.DefaultPDFTimeout)
}
}
// slow sends the headers, then the body in pieces over about 600 ms.
func slow(t *testing.T, ct string, first string) string {
t.Helper()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", ct)
w.WriteHeader(200)
fl, _ := w.(http.Flusher)
_, _ = w.Write([]byte(first))
if fl != nil {
fl.Flush()
}
for i := 0; i < 6; i++ {
time.Sleep(100 * time.Millisecond)
_, _ = w.Write(bytes.Repeat([]byte("y"), 1024))
if fl != nil {
fl.Flush()
}
}
}))
t.Cleanup(ts.Close)
return ts.URL
}
// The web-page deadline (Timeout) still applies to HTML; a PDF gets PDFTimeout.
func TestPDFDeadline(t *testing.T) {
f := loopbackFetcher()
f.Timeout = 250 * time.Millisecond
f.PDFTimeout = 3 * time.Second
if _, err := f.Fetch(context.Background(), slow(t, "application/pdf", "%PDF-1.7\n")); err != nil {
t.Errorf("slow pdf under PDFTimeout: %v", err)
}
if _, err := f.Fetch(context.Background(), slow(t, "application/octet-stream", "%PDF-1.7\n")); err != nil {
t.Errorf("slow sniffed pdf under PDFTimeout: %v", err)
}
start := time.Now()
if _, err := f.Fetch(context.Background(), slow(t, "text/html", "<html>")); err == nil {
t.Errorf("slow html past Timeout succeeded; want an error")
}
if d := time.Since(start); d > 2*time.Second {
t.Errorf("slow html took %v to fail; the 250 ms deadline was not applied", d)
}
}
func TestFetchIf(t *testing.T) {
var mu sync.Mutex
var got []string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
got = append(got, r.Header.Get("If-None-Match")+"|"+r.Header.Get("If-Modified-Since"))
mu.Unlock()
if r.Header.Get("If-None-Match") == `"v1"` {
w.WriteHeader(http.StatusNotModified)
return
}
w.Header().Set("Content-Type", "application/pdf")
w.Header().Set("ETag", `"v2"`)
w.Header().Set("Last-Modified", "Fri, 25 Sep 2026 12:00:00 GMT")
_, _ = w.Write([]byte("%PDF-1.7\nnew"))
}))
defer ts.Close()
f := loopbackFetcher()
res, err := f.FetchIf(context.Background(), ts.URL, `"v1"`, "")
if err != nil || res == nil || res.Status != http.StatusNotModified || len(res.Body) != 0 {
t.Fatalf("matching ETag: res %+v err %v; want status 304, empty body, nil error", res, err)
}
res, err = f.FetchIf(context.Background(), ts.URL, `"old"`, "Thu, 24 Sep 2026 12:00:00 GMT")
if err != nil || res.Status != 200 || string(res.Body) != "%PDF-1.7\nnew" || res.Header.Get("ETag") != `"v2"` {
t.Fatalf("changed document: res %+v err %v; want 200 with the new body and ETag", res, err)
}
mu.Lock()
sent := append([]string(nil), got...)
mu.Unlock() // never hold mu across another request: the handler takes it
if len(sent) != 2 || sent[0] != `"v1"|` || sent[1] != `"old"|Thu, 24 Sep 2026 12:00:00 GMT` {
t.Errorf("validators sent: %q", sent)
}
// With no validators FetchIf is a plain Fetch.
if _, err := f.FetchIf(context.Background(), ts.URL, "", ""); err != nil {
t.Errorf("FetchIf with no validators: %v", err)
}
}