kyleandClaude Opus 5.5
a804f5ac93
Owner review of p5c (tasks 01-02 by Ornith): - extract.go: an upload (up to 100 MiB) is read whole before the PDF pool's limits apply, so any number of concurrent uploads could buffer at once in a 1 GiB service. At most 2 bodies are read at a time now; a third waits up to 5 s, then 429 busy with Retry-After 10. - extract.go: `warnings` was null when empty; it is [] as in /v1/fetch. - extractcmd.go: extractURL replaced the client base URL's path; it now appends /v1/extract like fetch and search do. Regression tests in extract_review_test.go (api and cmd/sift). End-to-end (local serve -> Dixie; sift extract with SIFT_CLIENT): Llama 2 paper, 13.6 MB / 77 pages: --pages 1 ok, repeat served from the content-hash cache, --grep 2 hits, --question answered with page citations in 16.8 s. Log lines carry bytes/pages/mode/cached only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
43 lines
1.3 KiB
Go
43 lines
1.3 KiB
Go
package api
|
|
|
|
// P5c owner review: regression tests for /v1/extract.
|
|
|
|
import (
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Uploads are buffered whole (up to 100 MiB) before the PDF pool sees them, so the number read
|
|
// at once is bounded: a third concurrent upload waits, then is refused busy.
|
|
func TestExtractUploadsBounded(t *testing.T) {
|
|
r := newPDFRig(t)
|
|
for i := 0; i < cap(r.s.uploads); i++ {
|
|
r.s.uploads <- struct{}{}
|
|
}
|
|
start := time.Now()
|
|
code, out := r.extract(t, r.fleet, "", threePDF(), "application/pdf")
|
|
if code != http.StatusTooManyRequests {
|
|
t.Fatalf("upload with every slot taken: %d %v, want 429", code, out)
|
|
}
|
|
if d := time.Since(start); d < uploadWait-100*time.Millisecond {
|
|
t.Fatalf("refused after %v, want a wait of about %v first", d, uploadWait)
|
|
}
|
|
<-r.s.uploads
|
|
if code, out := r.extract(t, r.fleet, "", threePDF(), "application/pdf"); code != 200 {
|
|
t.Fatalf("upload with a free slot: %d %v", code, out)
|
|
}
|
|
}
|
|
|
|
// The envelope's warnings are a list, never null, as in /v1/fetch.
|
|
func TestExtractWarningsIsAList(t *testing.T) {
|
|
r := newPDFRig(t)
|
|
code, out := r.extract(t, r.fleet, "", threePDF(), "application/pdf")
|
|
if code != 200 {
|
|
t.Fatalf("%d %v", code, out)
|
|
}
|
|
if _, ok := out["warnings"].([]any); !ok {
|
|
t.Fatalf("warnings = %#v, want a list", out["warnings"])
|
|
}
|
|
}
|