P5a: digests cite pages; sift_pdf_* metrics; sift fetch --pages/--grep; skill, unit and README

Digest systemPrompt gains a page-cite instruction and PromptVersion is 3. Add the sift_pdf_* metric series (extract_total{outcome}, extract_seconds, pages_total, pool_wait_seconds, cache_hits_total) to internal/metrics and wire ObservePDF/ObservePDFWait/PDFCacheHit into internal/api/pdf.go. Refactor cmd/sift/fetchcmd.go to a fetchOpts struct with parseFetchOpts and fetchBody, adding --pages/--grep. Update skill/SKILL.md, set nix MemoryMax to 1G, and add a PDF documents section to the README. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
2026-09-26 04:36:49 -07:00
parent 09f738bcac
commit 7e78de633f
11 changed files with 391 additions and 41 deletions
@@ -91,7 +91,7 @@ func loadClient() (base, key string, err error) {
 }

 func fetchMain() {
-	rawURL, mode, jq, question, noCtx, recoverFlag, asJSON, err := parseFetchArgs(os.Args[1:])
+	o, err := parseFetchOpts(os.Args[1:])
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(2)
@@ -101,23 +101,7 @@ func fetchMain() {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(5)
 	}
-	modeVal := mode
-	if modeVal == "" {
-		if question != "" {
-			modeVal = "digest"
-		} else {
-			modeVal = "clean"
-		}
-	}
-	wantCtx := question != "" && !noCtx
-	body, _ := json.Marshal(map[string]any{
-		"url":      rawURL,
-		"mode":     modeVal,
-		"jq":       jq,
-		"question": question,
-		"context":  wantCtx,
-		"recover":  recoverFlag,
-	})
+	body, _ := json.Marshal(fetchBody(o))
 	req, err := http.NewRequest(http.MethodPost, base+"/v1/fetch", bytes.NewReader(body))
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -146,7 +130,7 @@ func fetchMain() {
 		}
 		os.Exit(exitFor(resp.StatusCode, msg))
 	}
-	if asJSON {
+	if o.JSON {
 		enc := json.NewEncoder(os.Stdout)
 		enc.SetIndent("", "  ")
 		_ = enc.Encode(env)
@@ -177,36 +161,94 @@ func fetchMain() {
 	}
 }

-const fetchUsage = "usage: sift fetch <url> [--question Q] [--mode clean|raw|digest] [--recover] [--no-context] [--jq FILTER] [--json]"
+const fetchUsage = "usage: sift fetch <url> [--question Q] [--mode clean|raw|digest] [--pages a-b] [--grep RE] [--recover] [--no-context] [--jq FILTER] [--json]"

-func parseFetchArgs(args []string) (rawURL, mode, jq, question string, noCtx, recoverFlag, asJSON bool, err error) {
+// fetchOpts is every flag `sift fetch` accepts plus the single positional URL.
+type fetchOpts struct {
+	URL, Mode, JQ, Question, Pages, Grep string
+	NoContext, Recover, JSON             bool
+}
+
+// parseFetchOpts parses `sift fetch` args into fetchOpts. It knows which flags
+// take a value; the URL is the one positional.
+func parseFetchOpts(args []string) (fetchOpts, error) {
 	flagArgs, pos, err := splitAroundPositional(args, fetchValueFlags)
 	if err != nil {
-		return "", "", "", "", false, false, false, err
+		return fetchOpts{}, err
 	}
 	fs := flag.NewFlagSet("fetch", flag.ContinueOnError)
 	fs.SetOutput(io.Discard)
 	modeF := fs.String("mode", "", "clean|raw|digest (default digest if --question)")
 	jqF := fs.String("jq", "", "jq filter (JSON responses)")
 	questionF := fs.String("question", "", "focus the digest")
+	pagesF := fs.String("pages", "", "only these pages of a PDF (a-b or a)")
+	grepF := fs.String("grep", "", "only lines matching RE in a PDF")
 	noCtxF := fs.Bool("no-context", false, "skip Honcho context")
 	recoverF := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall")
 	_ = fs.Bool("no-main", false, "do not isolate main content")
 	asJSONF := fs.Bool("json", false, "print full /v1/fetch envelope")
 	if err := fs.Parse(flagArgs); err != nil {
-		return "", "", "", "", false, false, false, err
+		return fetchOpts{}, err
 	}
 	if fs.NArg() != 0 {
-		return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage)
+		return fetchOpts{}, fmt.Errorf("%s", fetchUsage)
 	}
 	if len(pos) != 1 {
-		return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage)
+		return fetchOpts{}, fmt.Errorf("%s", fetchUsage)
 	}
-	return pos[0], *modeF, *jqF, *questionF, *noCtxF, *recoverF, *asJSONF, nil
+	return fetchOpts{
+		URL:       pos[0],
+		Mode:      *modeF,
+		JQ:        *jqF,
+		Question:  *questionF,
+		Pages:     *pagesF,
+		Grep:      *grepF,
+		NoContext: *noCtxF,
+		Recover:   *recoverF,
+		JSON:      *asJSONF,
+	}, nil
+}
+
+// parseFetchArgs is the 8-value form the existing callers and tests use; it
+// delegates to parseFetchOpts.
+func parseFetchArgs(args []string) (rawURL, mode, jq, question string, noCtx, recoverFlag, asJSON bool, err error) {
+	o, err := parseFetchOpts(args)
+	if err != nil {
+		return "", "", "", "", false, false, false, err
+	}
+	return o.URL, o.Mode, o.JQ, o.Question, o.NoContext, o.Recover, o.JSON, nil
+}
+
+// fetchBody builds the /v1/fetch request body. Mode defaults to digest when a
+// question is set, else clean. Pages and grep are sent only when set.
+func fetchBody(o fetchOpts) map[string]any {
+	mode := o.Mode
+	if mode == "" {
+		if o.Question != "" {
+			mode = "digest"
+		} else {
+			mode = "clean"
+		}
+	}
+	body := map[string]any{
+		"url":      o.URL,
+		"mode":     mode,
+		"jq":       o.JQ,
+		"question": o.Question,
+		"context":  o.Question != "" && !o.NoContext,
+		"recover":  o.Recover,
+	}
+	if o.Pages != "" {
+		body["pages"] = o.Pages
+	}
+	if o.Grep != "" {
+		body["grep"] = o.Grep
+	}
+	return body
 }

 var fetchValueFlags = map[string]bool{
-	"mode": true, "jq": true, "question": true,
+	"mode": true, "jq": true, "question": true, "pages": true, "grep": true,
 }

 func splitAroundPositional(args []string, valueFlags map[string]bool) (flagArgs, pos []string, err error) {
@@ -0,0 +1,44 @@
+package main
+
+// P5a task 07: `sift fetch --pages` and `--grep`.
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+func TestParseFetchOptsPDF(t *testing.T) {
+	o, err := parseFetchOpts([]string{"https://example.org/t.pdf", "--pages", "41-60", "--grep", "attention head"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o.URL != "https://example.org/t.pdf" || o.Pages != "41-60" || o.Grep != "attention head" {
+		t.Errorf("got %+v", o)
+	}
+	o, err = parseFetchOpts([]string{"--grep", "x", "--question", "why", "https://example.org/t.pdf"})
+	if err != nil || o.URL != "https://example.org/t.pdf" || o.Grep != "x" || o.Question != "why" {
+		t.Errorf("flags before the URL: %+v %v", o, err)
+	}
+	if !strings.Contains(fetchUsage, "--pages") || !strings.Contains(fetchUsage, "--grep") {
+		t.Errorf("usage lacks the new flags: %s", fetchUsage)
+	}
+}
+
+func TestFetchBodyPDFFields(t *testing.T) {
+	b, _ := json.Marshal(fetchBody(fetchOpts{URL: "https://example.org/a.pdf"}))
+	if strings.Contains(string(b), `"pages"`) || strings.Contains(string(b), `"grep"`) {
+		t.Errorf("unset pages/grep must not be sent: %s", b)
+	}
+	b, _ = json.Marshal(fetchBody(fetchOpts{URL: "https://example.org/a.pdf", Pages: "2-3", Grep: "fox"}))
+	var m map[string]any
+	_ = json.Unmarshal(b, &m)
+	if m["pages"] != "2-3" || m["grep"] != "fox" || m["url"] != "https://example.org/a.pdf" || m["mode"] != "clean" {
+		t.Errorf("body = %s", b)
+	}
+	b, _ = json.Marshal(fetchBody(fetchOpts{URL: "u", Question: "q"}))
+	_ = json.Unmarshal(b, &m)
+	if m["mode"] != "digest" || m["question"] != "q" || m["context"] != true {
+		t.Errorf("question body = %s (mode must default to digest, context on)", b)
+	}
+}