diff --git a/README.md b/README.md index 680377e..5ffb2c4 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,18 @@ with `blocked: …`. sift serve [-listen addr] [-config /etc/sift/config.toml] [-keys /var/lib/sift/keys.toml] [-deny-prefix CIDR]... sift fetch [--question q] [--mode clean|raw|digest] [--jq filter] [--pages a-b] [--grep re] [--no-context] [--recover] [--json] +sift extract [--pages a-b] [--grep re] [--question q] [--mode clean|digest] [--json] sift search "" [--k n] [--json] sift keys add --caller name [--workspace ws] [--out file] | list | revoke --caller name sift version ``` -`sift fetch` and `sift search` are clients: they read `url` and `key_file` from `client.toml` -(`$SIFT_CLIENT`, else `$HERMES_HOME/sift/client.toml`, else `~/.config/sift/client.toml`); the key -file must be mode 0600. The agent-facing guide is [`skill/SKILL.md`](skill/SKILL.md). +`sift fetch`, `sift extract` and `sift search` are clients: they read `url` and `key_file` from +`client.toml` (`$SIFT_CLIENT`, else `$HERMES_HOME/sift/client.toml`, else `~/.config/sift/client.toml`); +the key file must be mode 0600. `sift extract` uploads a local PDF to `/v1/extract` and prints the +answer exactly as `sift fetch` does for a PDF URL — `--pages`, `--grep` and `--question` behave the +same, and a file over 100 MiB is refused before the upload. The agent-facing guide is +[`skill/SKILL.md`](skill/SKILL.md). ## Configuration diff --git a/cmd/sift/extract_test.go b/cmd/sift/extract_test.go new file mode 100644 index 0000000..9c06bca --- /dev/null +++ b/cmd/sift/extract_test.go @@ -0,0 +1,46 @@ +package main + +// P5c task 02: `sift extract ` uploads a PDF to /v1/extract. + +import ( + "net/url" + "strings" + "testing" +) + +func TestParseExtractOpts(t *testing.T) { + o, err := parseExtractOpts([]string{"thesis.pdf", "--pages", "41-60", "--grep", "attention head"}) + if err != nil || o.File != "thesis.pdf" || o.Pages != "41-60" || o.Grep != "attention head" { + t.Errorf("%+v %v", o, err) + } + o, err = parseExtractOpts([]string{"--question", "what is the method?", "--json", "paper.pdf"}) + if err != nil || o.File != "paper.pdf" || o.Question != "what is the method?" || !o.JSON { + t.Errorf("flags first: %+v %v", o, err) + } + if _, err := parseExtractOpts(nil); err == nil { + t.Error("no file: want an error") + } + if _, err := parseExtractOpts([]string{"a.pdf", "b.pdf"}); err == nil { + t.Error("two files: want an error") + } + if !strings.Contains(extractUsage, "sift extract ") { + t.Errorf("usage %q", extractUsage) + } +} + +func TestExtractURL(t *testing.T) { + u, err := url.Parse(extractURL("https://sift.example:8443", extractOpts{File: "x.pdf", Pages: "2-3", Grep: "a b&c"})) + if err != nil { + t.Fatal(err) + } + if u.Path != "/v1/extract" || u.Query().Get("pages") != "2-3" || u.Query().Get("grep") != "a b&c" { + t.Errorf("url %s", u) + } + if u.Query().Has("question") || u.Query().Has("mode") { + t.Errorf("unset options must not be sent: %s", u) + } + u, _ = url.Parse(extractURL("https://sift.example:8443", extractOpts{File: "x.pdf", Question: "why?"})) + if u.Query().Get("question") != "why?" || u.Query().Get("mode") != "digest" { + t.Errorf("question url %s (mode must be digest with a question)", u) + } +} diff --git a/cmd/sift/extractcmd.go b/cmd/sift/extractcmd.go new file mode 100644 index 0000000..eeaad0a --- /dev/null +++ b/cmd/sift/extractcmd.go @@ -0,0 +1,158 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" +) + +// maxExtractUpload mirrors the server's /v1/extract cap (a file over 100 MiB is +// refused on upload with 413), so reject it before uploading. +const maxExtractUpload = 100 << 20 + +type extractOpts struct { + File, Pages, Grep, Question, Mode string + JSON bool +} + +const extractUsage = "usage: sift extract [--pages a-b] [--grep RE] [--question Q] [--mode clean|digest] [--json]" + +// parseExtractOpts parses `sift extract` args into extractOpts. It knows which +// flags take a value; the PDF path is the one positional. +func parseExtractOpts(args []string) (extractOpts, error) { + flagArgs, pos, err := splitAroundPositional(args, extractValueFlags) + if err != nil { + return extractOpts{}, err + } + fs := flag.NewFlagSet("extract", flag.ContinueOnError) + fs.SetOutput(io.Discard) + modeF := fs.String("mode", "", "clean|digest (default digest if --question)") + pagesF := fs.String("pages", "", "only these pages of a PDF (a-b or a)") + grepF := fs.String("grep", "", "only lines matching RE in a PDF") + questionF := fs.String("question", "", "focus the digest") + asJSONF := fs.Bool("json", false, "print full /v1/extract envelope") + if err := fs.Parse(flagArgs); err != nil { + return extractOpts{}, err + } + if fs.NArg() != 0 { + return extractOpts{}, fmt.Errorf("%s", extractUsage) + } + if len(pos) != 1 { + return extractOpts{}, fmt.Errorf("%s", extractUsage) + } + return extractOpts{ + File: pos[0], + Mode: *modeF, + Pages: *pagesF, + Grep: *grepF, + Question: *questionF, + JSON: *asJSONF, + }, nil +} + +var extractValueFlags = map[string]bool{ + "mode": true, "pages": true, "grep": true, "question": true, +} + +// extractURL builds the /v1/extract URL from base plus only the options that are +// set; mode defaults to digest when a question is given and no mode was. +func extractURL(base string, o extractOpts) string { + v := url.Values{} + if o.Pages != "" { + v.Set("pages", o.Pages) + } + if o.Grep != "" { + v.Set("grep", o.Grep) + } + if o.Question != "" { + v.Set("question", o.Question) + } + mode := o.Mode + if mode == "" && o.Question != "" { + mode = "digest" + } + if mode != "" { + v.Set("mode", mode) + } + u, err := url.Parse(base) + if err != nil { + return base + "/v1/extract?" + v.Encode() + } + u.Path = "/v1/extract" + u.RawQuery = v.Encode() + return u.String() +} + +func extractMain() { + o, err := parseExtractOpts(os.Args[1:]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + base, key, err := loadClient() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(5) + } + f, err := os.Open(o.File) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(5) + } + defer f.Close() + st, err := f.Stat() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(5) + } + if st.Size() > maxExtractUpload { + fmt.Fprintf(os.Stderr, "extract: %s is %d bytes, over the %d MB limit\n", o.File, st.Size(), maxExtractUpload>>20) + os.Exit(3) + } + req, err := http.NewRequest(http.MethodPost, extractURL(base, o), f) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(5) + } + req.ContentLength = st.Size() + req.Header.Set("Authorization", "Bearer "+key) + req.Header.Set("Content-Type", "application/pdf") + resp, err := http.DefaultClient.Do(req) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(5) + } + defer resp.Body.Close() + var env map[string]any + if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(5) + } + if resp.StatusCode != 200 { + msg, _ := env["error"].(string) + hint, _ := env["hint"].(string) + if hint != "" { + fmt.Fprintf(os.Stderr, "%s\nhint: %s\n", msg, hint) + } else { + fmt.Fprintln(os.Stderr, msg) + } + os.Exit(exitFor(resp.StatusCode, msg)) + } + if o.JSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + _ = enc.Encode(env) + return + } + if c, ok := env["content"].(string); ok { + fmt.Print(c) + if !strings.HasSuffix(c, "\n") { + fmt.Println() + } + } +} diff --git a/cmd/sift/main.go b/cmd/sift/main.go index 91d0b1f..b42c8fd 100644 --- a/cmd/sift/main.go +++ b/cmd/sift/main.go @@ -22,6 +22,8 @@ func main() { serveMain() case "fetch": fetchMain() + case "extract": + extractMain() case "search": searchMain() case "keys": @@ -45,6 +47,7 @@ func usage() { Commands: sift serve [flags] run the service sift fetch fetch via the service + sift extract extract a local PDF via the service sift search "" search via the service sift keys add|list|revoke sift version diff --git a/docs/implementer-log.md b/docs/implementer-log.md index c6fe6d0..2ac42a7 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -16,6 +16,7 @@ One row per task session that ends in a commit. Newest last. | p5a/07-api-cache-pages | 2026-09-26 | done | 1 | gate: ok | none | Implemented the PDF document cache with revalidation in `doFetch`: `pdfFromCache` (miss / fresh within `PDFTTL` / stale → `FetchIf` with 304→`TouchPDF`+answer-fresh / 200→return response with ok=false and no second GET) and `storePDF` after `extractPDF` (`ContentHash=res.SHA256`, ETag/Last-Modified from headers, FetchedAt=now, Meta=JSON of `doc.Meta`, Pages=`doc.Text`). `pages`/`grep` passed into `pdfdoc.Options` in `renderPDF`; `ErrBadPages`/`ErrBadGrep`→400 with the error's own text, non-PDF→400 "pages and grep apply to PDFs only" before the doc block. Question limit in digest mode: `n=len(digest.Split(pdfdoc.Full(doc), ChunkTokens))`, `n>PDFMaxQuestionChunks`→422 "document too long for question (n chunks); use grep or pages" before any digest call. Envelope `cached` set true for cache answers via the synthetic cache res's empty body (`len(res.Body)==0`); the one `doFetch` change beyond the rule-1 call was reusing that 200 response instead of a second `Fetch`. Added `fmt` import to api.go. Given file byte-identical; make gate ok. | | p5b/03-api | 2026-09-26 | done | 1 | gate: ok | none | The digest now answers long documents itself (task 02), so the API's refusal limit is only a safety cap: `DefaultPDFMaxQuestionChunks` 200→5000 in `internal/api/pdf.go` (the 422 message text unchanged, still "document too long for question (N chunks); use grep or pages"). In `v1fetch` (`internal/api/api.go`) the digest-meta map gains `"retrieval": dr.Retrieval` and `"embedded": dr.Embedded` next to `chunks_used`/`chunks_total`, only when the digest ran without a fallback. Copied the two given tests byte-identical from the plan — the replacement `pdf_test.go` is now the protected one and asserts `DefaultPDFMaxQuestionChunks == 5000`; `pdf_retrieval_test.go` asserts the first long-doc question returns `retrieval=bm25`, `embedded=0`. README PDF section: a long-doc question is answered first from BM25 (`retrieval: "bm25"`) while its chunks embed in the background and hybrid on later questions, plus the 5000-chunk cap. One sentence added to `skill/SKILL.md`. All api tests pass under `-race -count=3`; make gate ok; given files byte-identical. | | p5c/01-extract-api | 2026-09-26 | done | 1 | gate: ok | none | `POST /v1/extract` answers a PDF posted in the body like `/v1/fetch` answers the same PDF by URL, nothing fetched. `internal/api/extract.go`: auth via `caller`; `io.ReadAll` of the body (a `*http.MaxBytesError`→413 "too large (>N MB)", empty→400); `clean.Kind` must be "pdf" else 415; query `pages`/`grep`/`question`/`mode` (digest when a question is set, else clean; other→400). Content-addressed cache: `hash="sha256:"+HashBody(body)`, `key=PDFKey("upload:"+hash, callerClass)`; a `GetPDF` hit rebuilds the doc via `pdfFromEntry` with `cached:true` + `PDFCacheHit` and no child (bypasses the TTL/revalidation that `pdfFromCache` does, so identical content is extracted once); a miss builds a synthetic `fetch.Resource{URL:"upload",...}` and runs `extractPDF`+`storePDF`. Bad pages/grep→400 via `writeFetchErr`, the chunk limit→the same 422 as `v1fetch`, extraction errors→`pdfError` via `writeFetchErr`. Envelope mirrors the PDF fetch path (`content`/`mode`/`untrusted`/`warnings`/`sizes`/`timings`/`pdf`/`digest`); `source` is `{"kind":"upload","hash","bytes","cached","fetched_at"}` with no `url` key; no Honcho context for uploads; one log line "extract" with caller/bytes/pdf_pages/mode/cached. `internal/api/api.go` scoped to the rule-1 changes: `DefaultPDFMaxUpload` const (100 MiB), `PDFMaxUpload int64` Server field, the `POST /v1/extract` route in `New`, and `ServeHTTP` (per-path upload cap instead of the 64 KiB JSON limit, and `/v1/extract` added to the `requestDeadline` switch). The `pdfMaxUpload()` field-or-default resolver lives in extract.go. Given test byte-identical; all api tests pass under `-race -count=3`; make gate ok. | +| p5c/02-extract-cli | 2026-09-26 | done | 1 | gate: ok | none | `sift extract ` uploads a local PDF to `/v1/extract` and prints the answer exactly as `sift fetch` does. `cmd/sift/extractcmd.go`: `extractOpts` + `parseExtractOpts` (reuses `splitAroundPositional` with `extractValueFlags` mode/pages/grep/question, one positional file, `--json`), `extractURL` builds `base+"/v1/extract"` with `url.Values` set only for the options that are set and `mode=digest` when a question is given and no mode — values are URL-encoded so a grep like `a b&c` survives round-trip; `extractMain` reuses `loadClient`, `stat`s the file and refuses >100 MiB before uploading (exit 3, matching the server's 413→exitFor), then `http.NewRequest` with the `*os.File` as the streaming body and `ContentLength` set (no second in-memory copy), `Content-Type: application/pdf` + bearer, and the same error/exitFor path as fetch; success prints `content` (or the `--json` envelope). `main.go`: `case "extract"` and one help line. README: extract line in the CLI block plus a sentence; SKILL.md: a PDF-on-disk bullet in the PDFs section. Given test byte-identical; all cmd/sift tests pass under `-race -count=3`; make gate ok; `bin/sift extract` prints the usage line. | ## P5a review (owner, 2026-09-26) diff --git a/skill/SKILL.md b/skill/SKILL.md index 7ea19c4..6366e55 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -60,6 +60,9 @@ for blocked pages), `--json` (full envelope: content, source, context, sizes, wa PDFs work: `sift fetch` extracts them and returns the text. A short one comes back whole; a long one comes back as a map — an outline with page numbers, then the first pages. Then: +- A PDF on disk (an attachment, a download): `sift extract ` uploads it to `/v1/extract` + and prints the answer as `sift fetch` does, with the same `--pages`/`--grep`/`--question`. + Never run `pdftotext` yourself. - `sift fetch URL --pages 41-60` for a page range, `--grep 'pattern'` for matching lines, or `--question "…"` for a digest that cites each fact as `(p. N)`; `--question` works on long PDFs, and the first answer may be `bm25` (lexical) — ask again later for a sharper one.