clean, fetch: recognise PDFs; 100 MB / 120 s for PDFs; FetchIf revalidates
Recognise PDFs in clean.Kind (declared types and %PDF- sniffing on octet-stream/bodyless bodies), give PDF responses the 100 MB body cap and 120 s deadline while web pages keep 5 MB / 20 s, and add FetchIf, which sends If-None-Match / If-Modified-Since and maps a 304 to a Resource with Status 304 and an empty Body. Fetch and FetchIf share one doFetch path for guard, client, limits and error mapping. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
+39
-1
@@ -28,7 +28,7 @@ type Result struct {
|
||||
Title string
|
||||
Byline string
|
||||
Date string
|
||||
Kind string // html, json, text
|
||||
Kind string // html, json, text, pdf
|
||||
Warning string
|
||||
Trunc bool
|
||||
}
|
||||
@@ -57,6 +57,8 @@ func MediaKind(ct string) string {
|
||||
return "json"
|
||||
case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
|
||||
return "text"
|
||||
case ct == "application/pdf" || ct == "application/x-pdf":
|
||||
return "pdf"
|
||||
case ct == "":
|
||||
return "sniff"
|
||||
default:
|
||||
@@ -76,6 +78,42 @@ func Sniff(body []byte) string {
|
||||
return "text"
|
||||
}
|
||||
|
||||
// Kind classifies a fetched body: "html", "json", "text", "pdf" or "other". The
|
||||
// media type is lower-cased and its parameters dropped, as MediaKind does.
|
||||
func Kind(contentType string, body []byte) string {
|
||||
ct := strings.ToLower(strings.TrimSpace(contentType))
|
||||
if i := strings.IndexByte(ct, ';'); i >= 0 {
|
||||
ct = strings.TrimSpace(ct[:i])
|
||||
}
|
||||
switch {
|
||||
case ct == "application/pdf" || ct == "application/x-pdf":
|
||||
return "pdf"
|
||||
case ct == "text/html" || ct == "application/xhtml+xml":
|
||||
return "html"
|
||||
case ct == "application/json" || strings.HasSuffix(ct, "+json"):
|
||||
return "json"
|
||||
case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
|
||||
return "text"
|
||||
case ct == "application/octet-stream" || ct == "binary/octet-stream":
|
||||
if containsPDF(body) {
|
||||
return "pdf"
|
||||
}
|
||||
return "other"
|
||||
case ct == "":
|
||||
if containsPDF(body) {
|
||||
return "pdf"
|
||||
}
|
||||
return Sniff(body)
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
// containsPDF reports whether %PDF- appears within the first 1024 bytes.
|
||||
func containsPDF(body []byte) bool {
|
||||
return bytes.Contains(body[:min(len(body), 1024)], []byte("%PDF-"))
|
||||
}
|
||||
|
||||
func HTML(res *fetch.Resource, onlyMain bool, outCap int) (*Result, error) {
|
||||
if outCap <= 0 {
|
||||
outCap = defaultOutCap
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package clean_test
|
||||
|
||||
// P5a task 01: Kind decides which cleaner a fetched body gets, now including "pdf".
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/clean"
|
||||
)
|
||||
|
||||
func TestKind(t *testing.T) {
|
||||
pdf := []byte("%PDF-1.7\n%\xe2\xe3\xcf\xd3\n1 0 obj << >> endobj\n")
|
||||
junkThenPDF := append([]byte(strings.Repeat(" ", 900)), pdf...) // the spec allows leading junk
|
||||
lateMagic := append([]byte(strings.Repeat("x", 2000)), pdf...) // past the first 1024 bytes
|
||||
binary := []byte{0x50, 0x4b, 0x03, 0x04, 0x00, 0x00} // a zip
|
||||
for _, tc := range []struct {
|
||||
name, ct string
|
||||
body []byte
|
||||
want string
|
||||
}{
|
||||
{"pdf by type", "application/pdf", pdf, "pdf"},
|
||||
{"pdf by type with params", "Application/PDF; qs=0.9", pdf, "pdf"},
|
||||
{"x-pdf", "application/x-pdf", pdf, "pdf"},
|
||||
{"pdf type, body not checked", "application/pdf", []byte("oops"), "pdf"},
|
||||
{"octet-stream sniffed as pdf", "application/octet-stream", pdf, "pdf"},
|
||||
{"binary/octet-stream sniffed as pdf", "binary/octet-stream", pdf, "pdf"},
|
||||
{"no type sniffed as pdf", "", pdf, "pdf"},
|
||||
{"leading junk within 1024 bytes", "application/octet-stream", junkThenPDF, "pdf"},
|
||||
{"magic too late", "application/octet-stream", lateMagic, "other"},
|
||||
{"octet-stream that is not a pdf", "application/octet-stream", binary, "other"},
|
||||
{"html stays html even if it mentions %PDF-", "text/html; charset=utf-8", []byte("<html>%PDF-1.4</html>"), "html"},
|
||||
{"json", "application/json", []byte(`{"a":1}`), "json"},
|
||||
{"problem+json", "application/problem+json", []byte(`{}`), "json"},
|
||||
{"text", "text/plain", []byte("hello"), "text"},
|
||||
{"no type, html", "", []byte("<!doctype html><html></html>"), "html"},
|
||||
{"no type, json", "", []byte(`[1,2]`), "json"},
|
||||
{"no type, text", "", []byte("just words"), "text"},
|
||||
{"zip", "application/zip", binary, "other"},
|
||||
{"image", "image/png", []byte{0x89, 'P', 'N', 'G'}, "other"},
|
||||
} {
|
||||
if got := clean.Kind(tc.ct, tc.body); got != tc.want {
|
||||
t.Errorf("%s: Kind(%q, …) = %q, want %q", tc.name, tc.ct, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MediaKind keeps answering for callers that have no body: the pdf types are "pdf" there too.
|
||||
func TestMediaKindKnowsPDF(t *testing.T) {
|
||||
for ct, want := range map[string]string{"application/pdf": "pdf", "application/x-pdf": "pdf", "text/html": "html", "": "sniff"} {
|
||||
if got := clean.MediaKind(ct); got != want {
|
||||
t.Errorf("MediaKind(%q) = %q, want %q", ct, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
+7
-62
@@ -6,7 +6,6 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -62,6 +61,11 @@ type Fetcher struct {
|
||||
UA string
|
||||
MaxBody int64
|
||||
Timeout time.Duration
|
||||
// PDFMaxBody and PDFTimeout apply to a PDF response; 0 means the
|
||||
// DefaultPDF* values. They live here (not in pdf.go) because they are
|
||||
// fields on the Fetcher, whose base definition stays in fetch.go.
|
||||
PDFMaxBody int64 // 0 = DefaultPDFMaxBody
|
||||
PDFTimeout time.Duration // 0 = DefaultPDFTimeout
|
||||
|
||||
clientOnce sync.Once
|
||||
client *http.Client
|
||||
@@ -91,7 +95,7 @@ func (f *Fetcher) ua() string {
|
||||
func (f *Fetcher) httpClient() *http.Client {
|
||||
f.clientOnce.Do(func() {
|
||||
c := f.Guard.Client()
|
||||
c.Timeout = f.timeout()
|
||||
c.Timeout = f.clientTimeout()
|
||||
f.client = c
|
||||
})
|
||||
return f.client
|
||||
@@ -99,66 +103,7 @@ func (f *Fetcher) httpClient() *http.Client {
|
||||
|
||||
// Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard.
|
||||
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
|
||||
if f == nil || f.Guard == nil {
|
||||
return nil, errf(500, "", "internal error")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, f.timeout())
|
||||
defer cancel()
|
||||
|
||||
if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil {
|
||||
return nil, mapGuard(err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, errf(400, "", "invalid url")
|
||||
}
|
||||
req.Header.Set("User-Agent", f.ua())
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
|
||||
// no cookies, no auth, no Referer
|
||||
|
||||
resp, err := f.httpClient().Do(req)
|
||||
if err != nil {
|
||||
return nil, mapDial(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
limit := f.maxBody()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
|
||||
if err != nil {
|
||||
return nil, mapDial(err)
|
||||
}
|
||||
if int64(len(body)) > limit {
|
||||
return nil, errf(413, "", "too large (>%d MB decompressed)", limit>>20)
|
||||
}
|
||||
|
||||
final := rawURL
|
||||
if resp.Request != nil && resp.Request.URL != nil {
|
||||
final = resp.Request.URL.String()
|
||||
}
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
res := &Resource{
|
||||
URL: rawURL,
|
||||
FinalURL: final,
|
||||
Status: resp.StatusCode,
|
||||
ContentType: ct,
|
||||
Header: resp.Header.Clone(),
|
||||
Body: body,
|
||||
FetchedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusOK:
|
||||
return res, nil
|
||||
case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451:
|
||||
return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode)
|
||||
case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header):
|
||||
return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser")
|
||||
case resp.StatusCode >= 400:
|
||||
return res, errf(422, "", "upstream %d", resp.StatusCode)
|
||||
default:
|
||||
return res, nil
|
||||
}
|
||||
return f.doFetch(ctx, rawURL, "", "")
|
||||
}
|
||||
|
||||
func looksLikeChallenge(body []byte, h http.Header) bool {
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPDFMaxBody is the body cap for a PDF response: 100 MiB, versus the
|
||||
// 5 MiB web cap (DefaultMaxBody).
|
||||
DefaultPDFMaxBody = 100 << 20
|
||||
// DefaultPDFTimeout is the deadline for a PDF response: 120 s, versus the
|
||||
// 20 s web deadline (DefaultTimeout).
|
||||
DefaultPDFTimeout = 120 * time.Second
|
||||
)
|
||||
|
||||
// magicPDF marks the start of a PDF, found within the first sniffWindow bytes of
|
||||
// an octet-stream or a bodyless response.
|
||||
var magicPDF = []byte("%PDF-")
|
||||
|
||||
// sniffWindow is how many leading bytes are inspected to recognise a PDF whose
|
||||
// Content-Type does not say so.
|
||||
const sniffWindow = 1024
|
||||
|
||||
func (f *Fetcher) pdfMaxBody() int64 {
|
||||
if f.PDFMaxBody <= 0 {
|
||||
return DefaultPDFMaxBody
|
||||
}
|
||||
return f.PDFMaxBody
|
||||
}
|
||||
|
||||
func (f *Fetcher) pdfTimeout() time.Duration {
|
||||
if f.PDFTimeout <= 0 {
|
||||
return DefaultPDFTimeout
|
||||
}
|
||||
return f.PDFTimeout
|
||||
}
|
||||
|
||||
// clientTimeout is the http.Client.Timeout: it must cover the longest deadline
|
||||
// this Fetcher can give a request, so a PDF is not cut off at the web deadline.
|
||||
func (f *Fetcher) clientTimeout() time.Duration {
|
||||
t := f.timeout()
|
||||
if p := f.pdfTimeout(); p > t {
|
||||
return p
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// normalizeContentType lower-cases a media type and drops its parameters.
|
||||
func normalizeContentType(ct string) string {
|
||||
ct = strings.ToLower(strings.TrimSpace(ct))
|
||||
if i := strings.IndexByte(ct, ';'); i >= 0 {
|
||||
ct = strings.TrimSpace(ct[:i])
|
||||
}
|
||||
return ct
|
||||
}
|
||||
|
||||
// isPDFContentType reports whether a media type is unambiguously a PDF.
|
||||
func isPDFContentType(ct string) bool {
|
||||
ct = normalizeContentType(ct)
|
||||
return ct == "application/pdf" || ct == "application/x-pdf"
|
||||
}
|
||||
|
||||
// FetchIf GETs rawURL like Fetch but adds conditional-GET validators: a non-empty
|
||||
// etag is sent as If-None-Match and a non-empty lastModified as If-Modified-Since.
|
||||
// A 304 answer returns a Resource with Status 304, an empty Body and a nil error.
|
||||
// With both empty it is exactly Fetch.
|
||||
func (f *Fetcher) FetchIf(ctx context.Context, rawURL, etag, lastModified string) (*Resource, error) {
|
||||
return f.doFetch(ctx, rawURL, etag, lastModified)
|
||||
}
|
||||
|
||||
// doFetch is the shared body of Fetch and FetchIf: one guard check, one client,
|
||||
// one set of limits and error mappings, differing only in request headers.
|
||||
func (f *Fetcher) doFetch(ctx context.Context, rawURL, etag, lastModified string) (*Resource, error) {
|
||||
if f == nil || f.Guard == nil {
|
||||
return nil, errf(500, "", "internal error")
|
||||
}
|
||||
|
||||
// A PDF may take longer than a web page. Give the request the PDF deadline
|
||||
// and cancel it at the ordinary (web) deadline unless the response is
|
||||
// recognised as a PDF sooner, by headers or by the first bytes sniffed.
|
||||
pdfDeadline := f.pdfTimeout()
|
||||
ctx, cancel := context.WithTimeout(ctx, pdfDeadline)
|
||||
var webTimer *time.Timer
|
||||
if web := f.timeout(); web < pdfDeadline {
|
||||
webTimer = time.AfterFunc(web, cancel)
|
||||
}
|
||||
|
||||
if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil {
|
||||
cancel()
|
||||
return nil, mapGuard(err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, errf(400, "", "invalid url")
|
||||
}
|
||||
req.Header.Set("User-Agent", f.ua())
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
|
||||
if etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
if lastModified != "" {
|
||||
req.Header.Set("If-Modified-Since", lastModified)
|
||||
}
|
||||
// no cookies, no auth, no Referer
|
||||
|
||||
resp, err := f.httpClient().Do(req)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, mapDial(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// The headers are in; a declared PDF type no longer needs the web deadline.
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
pdfByHeader := isPDFContentType(ct)
|
||||
if pdfByHeader && webTimer != nil {
|
||||
webTimer.Stop()
|
||||
}
|
||||
|
||||
body, err := f.readBody(resp, ct, pdfByHeader, func() {
|
||||
// Sniffing an octet-stream or bodyless response as a PDF: keep the PDF
|
||||
// deadline instead of cancelling at the web deadline.
|
||||
if webTimer != nil {
|
||||
webTimer.Stop()
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
final := rawURL
|
||||
if resp.Request != nil && resp.Request.URL != nil {
|
||||
final = resp.Request.URL.String()
|
||||
}
|
||||
res := &Resource{
|
||||
URL: rawURL,
|
||||
FinalURL: final,
|
||||
Status: resp.StatusCode,
|
||||
ContentType: ct,
|
||||
Header: resp.Header.Clone(),
|
||||
Body: body,
|
||||
FetchedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusOK:
|
||||
cancel()
|
||||
return res, nil
|
||||
case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451:
|
||||
cancel()
|
||||
return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode)
|
||||
case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header):
|
||||
cancel()
|
||||
return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser")
|
||||
case resp.StatusCode == http.StatusNotModified:
|
||||
cancel()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return &Resource{
|
||||
URL: rawURL,
|
||||
FinalURL: final,
|
||||
Status: resp.StatusCode,
|
||||
ContentType: ct,
|
||||
Header: resp.Header.Clone(),
|
||||
}, nil
|
||||
case resp.StatusCode >= 400:
|
||||
cancel()
|
||||
return res, errf(422, "", "upstream %d", resp.StatusCode)
|
||||
default:
|
||||
cancel()
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
|
||||
// readBody reads resp.Body under the cap that applies and reports whether the
|
||||
// response is a PDF (by declared type or by sniffing an octet-stream/bodyless
|
||||
// body). onPDF, when non-nil, is called once the body is sniffed as a PDF so the
|
||||
// caller can drop the web deadline.
|
||||
func (f *Fetcher) readBody(resp *http.Response, ct string, pdfByHeader bool, onPDF func()) ([]byte, error) {
|
||||
norm := normalizeContentType(ct)
|
||||
if !pdfByHeader && norm != "application/octet-stream" && norm != "binary/octet-stream" && norm != "" {
|
||||
// A known non-PDF type: the web cap, no sniffing.
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, f.maxBody()+1))
|
||||
if err != nil {
|
||||
return nil, mapDial(err)
|
||||
}
|
||||
if int64(len(body)) > f.maxBody() {
|
||||
return nil, errf(413, "", "too large (>%d MB decompressed)", f.maxBody()>>20)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// PDF by declared type, or a type that may hide one: sniff the leading bytes
|
||||
// to pick the cap, then read under it.
|
||||
pdf := pdfByHeader
|
||||
cap := f.maxBody()
|
||||
if pdf {
|
||||
cap = f.pdfMaxBody()
|
||||
}
|
||||
var body []byte
|
||||
if !pdfByHeader {
|
||||
head, err := io.ReadAll(io.LimitReader(resp.Body, sniffWindow))
|
||||
if err != nil {
|
||||
return nil, mapDial(err)
|
||||
}
|
||||
if bytes.Contains(head, magicPDF) {
|
||||
pdf = true
|
||||
cap = f.pdfMaxBody()
|
||||
if onPDF != nil {
|
||||
onPDF()
|
||||
}
|
||||
}
|
||||
body = head
|
||||
}
|
||||
if remaining := cap + 1 - int64(len(body)); remaining > 0 {
|
||||
more, err := io.ReadAll(io.LimitReader(resp.Body, remaining))
|
||||
if err != nil {
|
||||
return nil, mapDial(err)
|
||||
}
|
||||
body = append(body, more...)
|
||||
}
|
||||
if int64(len(body)) > cap {
|
||||
return nil, errf(413, "", "too large (>%d MB decompressed)", cap>>20)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package fetch_test
|
||||
|
||||
// P5a task 01: a PDF response gets a larger body cap and a longer deadline than a web page, and
|
||||
// FetchIf revalidates a cached document with a conditional GET.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/fetch"
|
||||
"git.wntrmute.dev/kyle/sift/internal/guard"
|
||||
)
|
||||
|
||||
func loopbackFetcher() *fetch.Fetcher {
|
||||
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
||||
return &fetch.Fetcher{Guard: g, UA: "sift-test"}
|
||||
}
|
||||
|
||||
func body(prefix string, n int) []byte {
|
||||
b := bytes.Repeat([]byte("x"), n)
|
||||
copy(b, prefix)
|
||||
return b
|
||||
}
|
||||
|
||||
func serve(t *testing.T, ct string, b []byte) string {
|
||||
t.Helper()
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if ct != "" {
|
||||
w.Header().Set("Content-Type", ct)
|
||||
}
|
||||
_, _ = w.Write(b)
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
return ts.URL
|
||||
}
|
||||
|
||||
func status(err error) int {
|
||||
var fe *fetch.Error
|
||||
if errors.As(err, &fe) {
|
||||
return fe.Status
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func TestPDFBodyCap(t *testing.T) {
|
||||
six := 6 << 20 // over DefaultMaxBody (5 MiB), under DefaultPDFMaxBody (100 MiB)
|
||||
for _, tc := range []struct {
|
||||
name, ct string
|
||||
b []byte
|
||||
want int // 0 = ok
|
||||
}{
|
||||
{"pdf by type", "application/pdf", body("%PDF-1.7\n", six), 0},
|
||||
{"pdf by sniffing octet-stream", "application/octet-stream", body("%PDF-1.7\n", six), 0},
|
||||
{"pdf with no type", "", body("%PDF-1.7\n", six), 0},
|
||||
{"html keeps the 5 MiB cap", "text/html", body("<html>", six), 413},
|
||||
{"octet-stream that is not a pdf keeps the cap", "application/octet-stream", body("PK\x03\x04", six), 413},
|
||||
} {
|
||||
f := loopbackFetcher()
|
||||
res, err := f.Fetch(context.Background(), serve(t, tc.ct, tc.b))
|
||||
if tc.want == 0 {
|
||||
if err != nil || res == nil || len(res.Body) != six {
|
||||
t.Errorf("%s: err %v, body %d bytes; want the whole %d", tc.name, err, lenBody(res), six)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if status(err) != tc.want {
|
||||
t.Errorf("%s: err %v (status %d), want %d", tc.name, err, status(err), tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func lenBody(r *fetch.Resource) int {
|
||||
if r == nil {
|
||||
return -1
|
||||
}
|
||||
return len(r.Body)
|
||||
}
|
||||
|
||||
func TestPDFMaxBodyIsConfigurable(t *testing.T) {
|
||||
f := loopbackFetcher()
|
||||
f.PDFMaxBody = 1 << 20
|
||||
_, err := f.Fetch(context.Background(), serve(t, "application/pdf", body("%PDF-1.7\n", 2<<20)))
|
||||
if status(err) != 413 || !strings.Contains(err.Error(), "too large") {
|
||||
t.Errorf("2 MiB pdf over a 1 MiB PDFMaxBody: %v, want 413 too large", err)
|
||||
}
|
||||
if fetch.DefaultPDFMaxBody != 100<<20 || fetch.DefaultPDFTimeout != 120*time.Second {
|
||||
t.Errorf("defaults = %d, %v; want 100 MiB, 120s", fetch.DefaultPDFMaxBody, fetch.DefaultPDFTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// slow sends the headers, then the body in pieces over about 600 ms.
|
||||
func slow(t *testing.T, ct string, first string) string {
|
||||
t.Helper()
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", ct)
|
||||
w.WriteHeader(200)
|
||||
fl, _ := w.(http.Flusher)
|
||||
_, _ = w.Write([]byte(first))
|
||||
if fl != nil {
|
||||
fl.Flush()
|
||||
}
|
||||
for i := 0; i < 6; i++ {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
_, _ = w.Write(bytes.Repeat([]byte("y"), 1024))
|
||||
if fl != nil {
|
||||
fl.Flush()
|
||||
}
|
||||
}
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
return ts.URL
|
||||
}
|
||||
|
||||
// The web-page deadline (Timeout) still applies to HTML; a PDF gets PDFTimeout.
|
||||
func TestPDFDeadline(t *testing.T) {
|
||||
f := loopbackFetcher()
|
||||
f.Timeout = 250 * time.Millisecond
|
||||
f.PDFTimeout = 3 * time.Second
|
||||
if _, err := f.Fetch(context.Background(), slow(t, "application/pdf", "%PDF-1.7\n")); err != nil {
|
||||
t.Errorf("slow pdf under PDFTimeout: %v", err)
|
||||
}
|
||||
if _, err := f.Fetch(context.Background(), slow(t, "application/octet-stream", "%PDF-1.7\n")); err != nil {
|
||||
t.Errorf("slow sniffed pdf under PDFTimeout: %v", err)
|
||||
}
|
||||
start := time.Now()
|
||||
if _, err := f.Fetch(context.Background(), slow(t, "text/html", "<html>")); err == nil {
|
||||
t.Errorf("slow html past Timeout succeeded; want an error")
|
||||
}
|
||||
if d := time.Since(start); d > 2*time.Second {
|
||||
t.Errorf("slow html took %v to fail; the 250 ms deadline was not applied", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchIf(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var got []string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
got = append(got, r.Header.Get("If-None-Match")+"|"+r.Header.Get("If-Modified-Since"))
|
||||
mu.Unlock()
|
||||
if r.Header.Get("If-None-Match") == `"v1"` {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/pdf")
|
||||
w.Header().Set("ETag", `"v2"`)
|
||||
w.Header().Set("Last-Modified", "Fri, 25 Sep 2026 12:00:00 GMT")
|
||||
_, _ = w.Write([]byte("%PDF-1.7\nnew"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
f := loopbackFetcher()
|
||||
|
||||
res, err := f.FetchIf(context.Background(), ts.URL, `"v1"`, "")
|
||||
if err != nil || res == nil || res.Status != http.StatusNotModified || len(res.Body) != 0 {
|
||||
t.Fatalf("matching ETag: res %+v err %v; want status 304, empty body, nil error", res, err)
|
||||
}
|
||||
res, err = f.FetchIf(context.Background(), ts.URL, `"old"`, "Thu, 24 Sep 2026 12:00:00 GMT")
|
||||
if err != nil || res.Status != 200 || string(res.Body) != "%PDF-1.7\nnew" || res.Header.Get("ETag") != `"v2"` {
|
||||
t.Fatalf("changed document: res %+v err %v; want 200 with the new body and ETag", res, err)
|
||||
}
|
||||
mu.Lock()
|
||||
sent := append([]string(nil), got...)
|
||||
mu.Unlock() // never hold mu across another request: the handler takes it
|
||||
if len(sent) != 2 || sent[0] != `"v1"|` || sent[1] != `"old"|Thu, 24 Sep 2026 12:00:00 GMT` {
|
||||
t.Errorf("validators sent: %q", sent)
|
||||
}
|
||||
// With no validators FetchIf is a plain Fetch.
|
||||
if _, err := f.FetchIf(context.Background(), ts.URL, "", ""); err != nil {
|
||||
t.Errorf("FetchIf with no validators: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user