kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P5a plan: PDF documents — seven Ornith tasks with acceptance tests first

Browse Source 
docs/plans/p5a: 01 kind+fetch limits, 02 pdftext child + runner, 03 dcache PDF tables,
04 pdfpool, 05 pdfdoc render (map/pages/grep), 06 api, 07 digest citations, metrics, CLI,
skill, unit, README. Given tests compile against a panic-only skeleton and fail there for the
intended reasons; PDFs for tests are built by internal/pdftest (given), plus two qpdf-encrypted
fixtures.

Owner changes for the run: vendor go-pdfkit/extract v0.3.0 + reader v0.6.0 (pinned via
tools.go until task 02); AGENTS.md, tools/run-plan.sh (enforces a Co-Authored-By trailer for
the model), docs/implementer-log.md, `make gate`; gofmt (whitespace) on clean.go and guard;
TestUnsupportedType uses a zip now that PDFs are supported; requestDeadline 90→210 s and
writeTimeout 120→240 s for the PDF path's worst case.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kyle
2026-09-25 20:47:38 -07:00
co-authored by Claude Opus 5.5
parent 3f2b78d4aa
commit cfde35397a
115 changed files with 26422 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/api.go
+2 -2
View File
@@ -30,8 +30,8 @@ import (
const (
maxReqBody = 64 << 10
writeTimeout = 120 * time.Second
requestDeadline = 90 * time.Second
writeTimeout = 240 * time.Second // P5a: a PDF may take 120 s to download + 20 s queued + 60 s to extract
requestDeadline = 210 * time.Second // P5a: the PDF path's worst case (see writeTimeout); other paths keep their own shorter timeouts
)
type Server struct {
internal/api/api_test.go
+4 -3
View File
@@ -222,8 +222,9 @@ func TestJSONJQ(t *testing.T) {
func TestUnsupportedType(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/pdf")
_, _ = w.Write([]byte("%PDF"))
// P5a: a PDF is supported now; a zip is still not.
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write([]byte("PK\x03\x04"))
}))
defer ts.Close()
s, key := setupAPI(t, g)
@@ -260,7 +261,7 @@ func TestJQTooLarge(t *testing.T) {
func TestHTTPServerWriteTimeout(t *testing.T) {
s := New("test", &guard.Guard{}, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
hs := s.HTTPServer("127.0.0.1:0")
if hs.WriteTimeout != 120*time.Second {
if hs.WriteTimeout != 240*time.Second { // P5a: 120 s download + 20 s queue + 60 s extraction fit inside
t.Fatalf("WriteTimeout %s", hs.WriteTimeout)
}
}
internal/clean/clean.go
+1 -1
View File
@@ -11,8 +11,8 @@ import (
"time"
"unicode/utf8"
readability "github.com/go-shiori/go-readability"
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
readability "github.com/go-shiori/go-readability"
"github.com/itchyny/gojq"
"golang.org/x/net/html"
"golang.org/x/net/html/charset"
internal/guard/guard_test.go
+24 -24
View File
@@ -15,9 +15,9 @@ import (
type nopConn struct{}
func (nopConn) Control(func(uintptr)) error { return nil }
func (nopConn) Read(func(uintptr) bool) error { return nil }
func (nopConn) Write(func(uintptr) bool) error { return nil }
func (nopConn) Control(func(uintptr)) error { return nil }
func (nopConn) Read(func(uintptr) bool) error { return nil }
func (nopConn) Write(func(uintptr) bool) error { return nil }
func mustAddr(s string) netip.Addr {
a, err := netip.ParseAddr(s)
@@ -38,10 +38,10 @@ func TestCheckAddrTable(t *testing.T) {
{"1.1.1.1", false, ""},
{"93.184.216.34", false, ""},
{"2001:4860:4860::8888", false, ""},
{"172.15.255.255", false, ""}, // just below 172.16/12
{"172.32.0.1", false, ""}, // just above 172.16/12
{"100.63.255.255", false, ""}, // just below CGNAT
{"100.128.0.1", false, ""}, // just above CGNAT
{"172.15.255.255", false, ""}, // just below 172.16/12
{"172.32.0.1", false, ""}, // just above 172.16/12
{"100.63.255.255", false, ""}, // just below CGNAT
{"100.128.0.1", false, ""}, // just above CGNAT
{"127.0.0.1", true, "loopback"},
{"127.255.255.254", true, "loopback"},
@@ -147,13 +147,13 @@ func TestDecimalOctalHexIPs(t *testing.T) {
// All of these are 127.0.0.1 in inet_aton form.
loopbacks := []string{
"http://127.0.0.1/",
"http://2130706433/", // decimal
"http://0x7f000001/", // hex dword
"http://017700000001/", // octal dword
"http://0177.0.0.1/", // octal dotted
"http://0x7f.0.0.1/", // hex dotted
"http://127.1/", // short
"http://127.0.1/", // 3-part
"http://2130706433/", // decimal
"http://0x7f000001/", // hex dword
"http://017700000001/", // octal dword
"http://0177.0.0.1/", // octal dotted
"http://0x7f.0.0.1/", // hex dotted
"http://127.1/", // short
"http://127.0.1/", // 3-part
"http://0x7f.1/",
}
g := &Guard{}
@@ -477,9 +477,9 @@ func mustURL(s string) *url.URL {
// Ensure Control implements the Dialer hook shape used by net.Dialer.
var (
_ func(network, address string, c syscall.RawConn) error = (*Guard)(nil).Control
_ func(ctx context.Context, network, address string, c syscall.RawConn) error = (*Guard)(nil).ControlContext
_ func(req *http.Request, via []*http.Request) error = (*Guard)(nil).CheckRedirect
_ func(network, address string, c syscall.RawConn) error = (*Guard)(nil).Control
_ func(ctx context.Context, network, address string, c syscall.RawConn) error = (*Guard)(nil).ControlContext
_ func(req *http.Request, via []*http.Request) error = (*Guard)(nil).CheckRedirect
)
func TestLocalhostName(t *testing.T) {
@@ -544,20 +544,20 @@ func TestIANASpecialPurpose(t *testing.T) {
t.Parallel()
// One address per required prefix from the P0 review.
blocked := []string{
"2002:a00:1::1", // 6to4 of 10.0.0.1
"2002:7f00:1::1", // 6to4 of 127.0.0.1
"2001::1", // Teredo 2001::/32 (inside 2001::/23)
"2002:a00:1::1", // 6to4 of 10.0.0.1
"2002:7f00:1::1", // 6to4 of 127.0.0.1
"2001::1", // Teredo 2001::/32 (inside 2001::/23)
"64:ff9b:1::a00:1", // RFC 8215 local-use NAT64 embedding 10.0.0.1
"fec0::1", // deprecated site-local
"198.18.0.1", // benchmarking
"fec0::1", // deprecated site-local
"198.18.0.1", // benchmarking
"198.19.255.255",
"192.0.0.1",
"240.0.0.1",
"255.255.255.255",
"100::1",
"2001:db8::1",
"2001:10::1", // ORCHID
"2001:20::1", // ORCHIDv2
"2001:10::1", // ORCHID
"2001:20::1", // ORCHIDv2
}
for _, s := range blocked {
t.Run(s, func(t *testing.T) {
internal/guard/iana.go
+3 -2
View File
@@ -7,8 +7,9 @@ import "net/netip"
// rather than decoded. IsGlobalUnicast remains a final gate in CheckAddr.
//
// Sources:
// https://www.iana.org/assignments/iana-ipv4-special-registry
// https://www.iana.org/assignments/iana-ipv6-special-registry
//
// https://www.iana.org/assignments/iana-ipv4-special-registry
// https://www.iana.org/assignments/iana-ipv6-special-registry
var ianaDeny = []struct {
prefix netip.Prefix
reason string