cli: sift extract uploads a local PDF; README and skill

`sift extract <file.pdf>` stats the file (refusing over 100 MiB before the
upload), streams it as the POST body with Content-Type application/pdf and the
ContentLength set, and prints the /v1/extract answer exactly as `sift fetch`
prints a PDF URL — same content output, same errors and exitFor codes.
parseExtractOpts reuses splitAroundPositional; extractURL sends only the set
options with mode=digest when a question is given. main.go gains the case and a
help line; README and skill document the command.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

README.md

@@ -33,14 +33,18 @@ with `blocked: …`.
 sift serve  [-listen addr] [-config /etc/sift/config.toml] [-keys /var/lib/sift/keys.toml] [-deny-prefix CIDR]...
 sift fetch  <url> [--question q] [--mode clean|raw|digest] [--jq filter] [--pages a-b] [--grep re]
                   [--no-context] [--recover] [--json]
+sift extract <file.pdf> [--pages a-b] [--grep re] [--question q] [--mode clean|digest] [--json]
 sift search "<query>" [--k n] [--json]
 sift keys   add --caller name [--workspace ws] [--out file] | list | revoke --caller name
 sift version
 ```
 
-`sift fetch` and `sift search` are clients: they read `url` and `key_file` from `client.toml`
+`sift fetch`, `sift extract` and `sift search` are clients: they read `url` and `key_file` from
-(`$SIFT_CLIENT`, else `$HERMES_HOME/sift/client.toml`, else `~/.config/sift/client.toml`); the key
+`client.toml` (`$SIFT_CLIENT`, else `$HERMES_HOME/sift/client.toml`, else `~/.config/sift/client.toml`);
-file must be mode 0600. The agent-facing guide is [`skill/SKILL.md`](skill/SKILL.md).
+the key file must be mode 0600. `sift extract` uploads a local PDF to `/v1/extract` and prints the
+answer exactly as `sift fetch` does for a PDF URL — `--pages`, `--grep` and `--question` behave the
+same, and a file over 100 MiB is refused before the upload. The agent-facing guide is
+[`skill/SKILL.md`](skill/SKILL.md).
 
 ## Configuration
 

cmd/sift/extract_test.go

@@ -0,0 +1,46 @@
+package main
+
+// P5c task 02: `sift extract <file>` uploads a PDF to /v1/extract.
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestParseExtractOpts(t *testing.T) {
+	o, err := parseExtractOpts([]string{"thesis.pdf", "--pages", "41-60", "--grep", "attention head"})
+	if err != nil || o.File != "thesis.pdf" || o.Pages != "41-60" || o.Grep != "attention head" {
+		t.Errorf("%+v %v", o, err)
+	}
+	o, err = parseExtractOpts([]string{"--question", "what is the method?", "--json", "paper.pdf"})
+	if err != nil || o.File != "paper.pdf" || o.Question != "what is the method?" || !o.JSON {
+		t.Errorf("flags first: %+v %v", o, err)
+	}
+	if _, err := parseExtractOpts(nil); err == nil {
+		t.Error("no file: want an error")
+	}
+	if _, err := parseExtractOpts([]string{"a.pdf", "b.pdf"}); err == nil {
+		t.Error("two files: want an error")
+	}
+	if !strings.Contains(extractUsage, "sift extract <file.pdf>") {
+		t.Errorf("usage %q", extractUsage)
+	}
+}
+
+func TestExtractURL(t *testing.T) {
+	u, err := url.Parse(extractURL("https://sift.example:8443", extractOpts{File: "x.pdf", Pages: "2-3", Grep: "a b&c"}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if u.Path != "/v1/extract" || u.Query().Get("pages") != "2-3" || u.Query().Get("grep") != "a b&c" {
+		t.Errorf("url %s", u)
+	}
+	if u.Query().Has("question") || u.Query().Has("mode") {
+		t.Errorf("unset options must not be sent: %s", u)
+	}
+	u, _ = url.Parse(extractURL("https://sift.example:8443", extractOpts{File: "x.pdf", Question: "why?"}))
+	if u.Query().Get("question") != "why?" || u.Query().Get("mode") != "digest" {
+		t.Errorf("question url %s (mode must be digest with a question)", u)
+	}
+}

cmd/sift/extractcmd.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+// maxExtractUpload mirrors the server's /v1/extract cap (a file over 100 MiB is
+// refused on upload with 413), so reject it before uploading.
+const maxExtractUpload = 100 << 20
+
+type extractOpts struct {
+	File, Pages, Grep, Question, Mode string
+	JSON                              bool
+}
+
+const extractUsage = "usage: sift extract <file.pdf> [--pages a-b] [--grep RE] [--question Q] [--mode clean|digest] [--json]"
+
+// parseExtractOpts parses `sift extract` args into extractOpts. It knows which
+// flags take a value; the PDF path is the one positional.
+func parseExtractOpts(args []string) (extractOpts, error) {
+	flagArgs, pos, err := splitAroundPositional(args, extractValueFlags)
+	if err != nil {
+		return extractOpts{}, err
+	}
+	fs := flag.NewFlagSet("extract", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	modeF := fs.String("mode", "", "clean|digest (default digest if --question)")
+	pagesF := fs.String("pages", "", "only these pages of a PDF (a-b or a)")
+	grepF := fs.String("grep", "", "only lines matching RE in a PDF")
+	questionF := fs.String("question", "", "focus the digest")
+	asJSONF := fs.Bool("json", false, "print full /v1/extract envelope")
+	if err := fs.Parse(flagArgs); err != nil {
+		return extractOpts{}, err
+	}
+	if fs.NArg() != 0 {
+		return extractOpts{}, fmt.Errorf("%s", extractUsage)
+	}
+	if len(pos) != 1 {
+		return extractOpts{}, fmt.Errorf("%s", extractUsage)
+	}
+	return extractOpts{
+		File:     pos[0],
+		Mode:     *modeF,
+		Pages:    *pagesF,
+		Grep:     *grepF,
+		Question: *questionF,
+		JSON:     *asJSONF,
+	}, nil
+}
+
+var extractValueFlags = map[string]bool{
+	"mode": true, "pages": true, "grep": true, "question": true,
+}
+
+// extractURL builds the /v1/extract URL from base plus only the options that are
+// set; mode defaults to digest when a question is given and no mode was.
+func extractURL(base string, o extractOpts) string {
+	v := url.Values{}
+	if o.Pages != "" {
+		v.Set("pages", o.Pages)
+	}
+	if o.Grep != "" {
+		v.Set("grep", o.Grep)
+	}
+	if o.Question != "" {
+		v.Set("question", o.Question)
+	}
+	mode := o.Mode
+	if mode == "" && o.Question != "" {
+		mode = "digest"
+	}
+	if mode != "" {
+		v.Set("mode", mode)
+	}
+	u, err := url.Parse(base)
+	if err != nil {
+		return base + "/v1/extract?" + v.Encode()
+	}
+	u.Path = "/v1/extract"
+	u.RawQuery = v.Encode()
+	return u.String()
+}
+
+func extractMain() {
+	o, err := parseExtractOpts(os.Args[1:])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(2)
+	}
+	base, key, err := loadClient()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	f, err := os.Open(o.File)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	defer f.Close()
+	st, err := f.Stat()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	if st.Size() > maxExtractUpload {
+		fmt.Fprintf(os.Stderr, "extract: %s is %d bytes, over the %d MB limit\n", o.File, st.Size(), maxExtractUpload>>20)
+		os.Exit(3)
+	}
+	req, err := http.NewRequest(http.MethodPost, extractURL(base, o), f)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	req.ContentLength = st.Size()
+	req.Header.Set("Authorization", "Bearer "+key)
+	req.Header.Set("Content-Type", "application/pdf")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	defer resp.Body.Close()
+	var env map[string]any
+	if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	if resp.StatusCode != 200 {
+		msg, _ := env["error"].(string)
+		hint, _ := env["hint"].(string)
+		if hint != "" {
+			fmt.Fprintf(os.Stderr, "%s\nhint: %s\n", msg, hint)
+		} else {
+			fmt.Fprintln(os.Stderr, msg)
+		}
+		os.Exit(exitFor(resp.StatusCode, msg))
+	}
+	if o.JSON {
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		_ = enc.Encode(env)
+		return
+	}
+	if c, ok := env["content"].(string); ok {
+		fmt.Print(c)
+		if !strings.HasSuffix(c, "\n") {
+			fmt.Println()
+		}
+	}
+}

cmd/sift/main.go

@@ -22,6 +22,8 @@ func main() {
 		serveMain()
 	case "fetch":
 		fetchMain()
+	case "extract":
+		extractMain()
 	case "search":
 		searchMain()
 	case "keys":
@@ -45,6 +47,7 @@ func usage() {
 Commands:
   sift serve    [flags]     run the service
   sift fetch    <url>       fetch via the service
+  sift extract  <file.pdf>  extract a local PDF via the service
   sift search   "<query>"   search via the service
   sift keys     add|list|revoke
   sift version

docs/implementer-log.md

@@ -16,6 +16,7 @@ One row per task session that ends in a commit. Newest last.
 | p5a/07-api-cache-pages | 2026-09-26 | done | 1 | gate: ok | none | Implemented the PDF document cache with revalidation in `doFetch`: `pdfFromCache` (miss / fresh within `PDFTTL` / stale → `FetchIf` with 304→`TouchPDF`+answer-fresh / 200→return response with ok=false and no second GET) and `storePDF` after `extractPDF` (`ContentHash=res.SHA256`, ETag/Last-Modified from headers, FetchedAt=now, Meta=JSON of `doc.Meta`, Pages=`doc.Text`). `pages`/`grep` passed into `pdfdoc.Options` in `renderPDF`; `ErrBadPages`/`ErrBadGrep`→400 with the error's own text, non-PDF→400 "pages and grep apply to PDFs only" before the doc block. Question limit in digest mode: `n=len(digest.Split(pdfdoc.Full(doc), ChunkTokens))`, `n>PDFMaxQuestionChunks`→422 "document too long for question (n chunks); use grep or pages" before any digest call. Envelope `cached` set true for cache answers via the synthetic cache res's empty body (`len(res.Body)==0`); the one `doFetch` change beyond the rule-1 call was reusing that 200 response instead of a second `Fetch`. Added `fmt` import to api.go. Given file byte-identical; make gate ok. |
 | p5b/03-api | 2026-09-26 | done | 1 | gate: ok | none | The digest now answers long documents itself (task 02), so the API's refusal limit is only a safety cap: `DefaultPDFMaxQuestionChunks` 200→5000 in `internal/api/pdf.go` (the 422 message text unchanged, still "document too long for question (N chunks); use grep or pages"). In `v1fetch` (`internal/api/api.go`) the digest-meta map gains `"retrieval": dr.Retrieval` and `"embedded": dr.Embedded` next to `chunks_used`/`chunks_total`, only when the digest ran without a fallback. Copied the two given tests byte-identical from the plan — the replacement `pdf_test.go` is now the protected one and asserts `DefaultPDFMaxQuestionChunks == 5000`; `pdf_retrieval_test.go` asserts the first long-doc question returns `retrieval=bm25`, `embedded=0`. README PDF section: a long-doc question is answered first from BM25 (`retrieval: "bm25"`) while its chunks embed in the background and hybrid on later questions, plus the 5000-chunk cap. One sentence added to `skill/SKILL.md`. All api tests pass under `-race -count=3`; make gate ok; given files byte-identical. |
 | p5c/01-extract-api | 2026-09-26 | done | 1 | gate: ok | none | `POST /v1/extract` answers a PDF posted in the body like `/v1/fetch` answers the same PDF by URL, nothing fetched. `internal/api/extract.go`: auth via `caller`; `io.ReadAll` of the body (a `*http.MaxBytesError`→413 "too large (>N MB)", empty→400); `clean.Kind` must be "pdf" else 415; query `pages`/`grep`/`question`/`mode` (digest when a question is set, else clean; other→400). Content-addressed cache: `hash="sha256:"+HashBody(body)`, `key=PDFKey("upload:"+hash, callerClass)`; a `GetPDF` hit rebuilds the doc via `pdfFromEntry` with `cached:true` + `PDFCacheHit` and no child (bypasses the TTL/revalidation that `pdfFromCache` does, so identical content is extracted once); a miss builds a synthetic `fetch.Resource{URL:"upload",...}` and runs `extractPDF`+`storePDF`. Bad pages/grep→400 via `writeFetchErr`, the chunk limit→the same 422 as `v1fetch`, extraction errors→`pdfError` via `writeFetchErr`. Envelope mirrors the PDF fetch path (`content`/`mode`/`untrusted`/`warnings`/`sizes`/`timings`/`pdf`/`digest`); `source` is `{"kind":"upload","hash","bytes","cached","fetched_at"}` with no `url` key; no Honcho context for uploads; one log line "extract" with caller/bytes/pdf_pages/mode/cached. `internal/api/api.go` scoped to the rule-1 changes: `DefaultPDFMaxUpload` const (100 MiB), `PDFMaxUpload int64` Server field, the `POST /v1/extract` route in `New`, and `ServeHTTP` (per-path upload cap instead of the 64 KiB JSON limit, and `/v1/extract` added to the `requestDeadline` switch). The `pdfMaxUpload()` field-or-default resolver lives in extract.go. Given test byte-identical; all api tests pass under `-race -count=3`; make gate ok. |
+| p5c/02-extract-cli | 2026-09-26 | done | 1 | gate: ok | none | `sift extract <file.pdf>` uploads a local PDF to `/v1/extract` and prints the answer exactly as `sift fetch` does. `cmd/sift/extractcmd.go`: `extractOpts` + `parseExtractOpts` (reuses `splitAroundPositional` with `extractValueFlags` mode/pages/grep/question, one positional file, `--json`), `extractURL` builds `base+"/v1/extract"` with `url.Values` set only for the options that are set and `mode=digest` when a question is given and no mode — values are URL-encoded so a grep like `a b&c` survives round-trip; `extractMain` reuses `loadClient`, `stat`s the file and refuses >100 MiB before uploading (exit 3, matching the server's 413→exitFor), then `http.NewRequest` with the `*os.File` as the streaming body and `ContentLength` set (no second in-memory copy), `Content-Type: application/pdf` + bearer, and the same error/exitFor path as fetch; success prints `content` (or the `--json` envelope). `main.go`: `case "extract"` and one help line. README: extract line in the CLI block plus a sentence; SKILL.md: a PDF-on-disk bullet in the PDFs section. Given test byte-identical; all cmd/sift tests pass under `-race -count=3`; make gate ok; `bin/sift extract` prints the usage line. |
 
 ## P5a review (owner, 2026-09-26)
 

skill/SKILL.md

@@ -60,6 +60,9 @@ for blocked pages), `--json` (full envelope: content, source, context, sizes, wa
 PDFs work: `sift fetch` extracts them and returns the text. A short one comes back whole; a
 long one comes back as a map — an outline with page numbers, then the first pages. Then:
 
+- A PDF on disk (an attachment, a download): `sift extract <file.pdf>` uploads it to `/v1/extract`
+  and prints the answer as `sift fetch` does, with the same `--pages`/`--grep`/`--question`.
+  Never run `pdftotext` yourself.
 - `sift fetch URL --pages 41-60` for a page range, `--grep 'pattern'` for matching lines, or
   `--question "…"` for a digest that cites each fact as `(p. N)`; `--question` works on long
   PDFs, and the first answer may be `bm25` (lexical) — ask again later for a sharper one.