From c89281200ac8aafc8ca402621813e0bcba8ef7bc Mon Sep 17 00:00:00 2001 From: "K. Isom" Date: Sat, 26 Sep 2026 10:21:51 -0700 Subject: [PATCH] docs: bring README, skill and dist up to date through P5a README gains API, CLI and configuration sections and the PDF range/grep/question limits; the skill covers PDF busy and failure results; dist/sift.service matches the Nix unit (MemoryMax=1G, secret.env); the example config drops the unused [upstreams] block; stale task comments in api/pdf.go. Gate overridden by the owner: TestGrepLimits (given test) exceeds its 1 s budget under -race on this host. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++- dist/sift.service | 3 ++- dist/sift.toml.example | 10 +++------ internal/api/pdf.go | 5 +++-- skill/SKILL.md | 6 +++++ 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c2f1e59..e0775eb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,51 @@ LLM-oriented fetch and search for the Hermes fleet: fetch a resource and get it compact, and with useful context attached — using local models (Dixie), SearXNG and Brave on orion, and Honcho for retrieval. -Status: P1 clean fetch (`/v2/scrape`, `/v1/fetch`, CLI, auth). See [`docs/PLAN.md`](docs/PLAN.md) and [`docs/P1.md`](docs/P1.md). +Status: implemented through P5a: P1 clean fetch (`/v2/scrape`, `/v1/fetch`, CLI, auth), P2 +digest, fleet context and recover, P3 search, P2.7 Prometheus metrics, P5a PDF documents. The design +is [`docs/PLAN.md`](docs/PLAN.md); each phase has its own doc (`docs/P1.md` … `docs/P5a.md`). + +## API + +All endpoints except `/healthz` take `Authorization: Bearer ` (per caller; see `sift keys`). +The server binds `127.0.0.1:8890` and is published on the tailnet by `tailscale serve` at +`https://orion.scylla-hammerhead.ts.net:8443`. + +| Endpoint | What | +|---|---| +| `POST /v2/scrape` | Firecrawl-compatible fetch (`url`, `formats`) — what Hermes `web_extract` calls | +| `POST /v2/search` | Firecrawl-compatible search (`query`, `limit`) — what Hermes `web_search` calls | +| `POST /v1/fetch` | native fetch: `url`, `mode` (`clean`/`raw`/`digest`), `question`, `jq`, `context`, `recover`, and for PDFs `pages`, `grep` | +| `POST /v1/search` | native search: `query`, `question`, `k`, `rerank`, `digest` | +| `GET /metrics`, `/metrics.json` | Prometheus text or JSON; open on loopback, bearer elsewhere | +| `GET /healthz` | liveness | + +Every outbound connection goes through the SSRF guard (`internal/guard`): private, tailnet and +configured `deny_prefixes` addresses, and the host's own WAN address (`[self_ip]`), are refused +with `blocked: …`. + +## CLI + +``` +sift serve [-listen addr] [-config /etc/sift/config.toml] [-keys /var/lib/sift/keys.toml] [-deny-prefix CIDR]... +sift fetch [--question q] [--mode clean|raw|digest] [--jq filter] [--pages a-b] [--grep re] + [--no-context] [--recover] [--json] +sift search "" [--k n] [--json] +sift keys add --caller name [--workspace ws] [--out file] | list | revoke --caller name +sift version +``` + +`sift fetch` and `sift search` are clients: they read `url` and `key_file` from `client.toml` +(`$SIFT_CLIENT`, else `$HERMES_HOME/sift/client.toml`, else `~/.config/sift/client.toml`); the key +file must be mode 0600. The agent-facing guide is [`skill/SKILL.md`](skill/SKILL.md). + +## Configuration + +`/etc/sift/config.toml` (`$SIFT_CONFIG`); every key is shown in +[`dist/sift.toml.example`](dist/sift.toml.example). Paid-search keys (`BRAVE_API_KEY`, +`TAVILY_API_KEY`) live on the box in `/var/lib/sift/secret.env` (`$SIFT_SECRET`), never in git or +the Nix store. On NixOS use `nixosModules.sift` (`services.sift`), which writes the config and +the hardened unit. ## Build @@ -42,6 +86,11 @@ Answers (through `/v1/fetch` or `/v2/scrape`): - `--pages 41-60` for a range, `--grep 'pattern'` for matching lines with page context; - `--question "…"` for a **digest** that, on a PDF, cites each fact as `(p. N)`. +A range returns at most 100 pages; grep shows at most 100 matches and takes a pattern of at most +200 bytes. `pages` and `grep` on anything but a PDF are a `400`. A question over a document too long +to digest (more than 200 chunks) is refused with +`422 document too long for question (N chunks); use grep or pages`. + Scanned PDFs have no text layer and return `no text layer (scanned PDF?)`; point them at an OCR source rather than fixing them here. diff --git a/dist/sift.service b/dist/sift.service index c39dbbe..9d62a98 100644 --- a/dist/sift.service +++ b/dist/sift.service @@ -28,7 +28,7 @@ RestrictRealtime=true RestrictSUIDSGID=true LockPersonality=true MemoryDenyWriteExecute=true -MemoryMax=512M +MemoryMax=1G SystemCallArchitectures=native SystemCallFilter=@system-service SystemCallFilter=~@privileged @@ -37,6 +37,7 @@ AmbientCapabilities= ReadWritePaths=/var/lib/sift ConfigurationDirectory=sift StateDirectory=sift +EnvironmentFile=-/var/lib/sift/secret.env [Install] WantedBy=multi-user.target diff --git a/dist/sift.toml.example b/dist/sift.toml.example index edbc904..c0b426c 100644 --- a/dist/sift.toml.example +++ b/dist/sift.toml.example @@ -1,5 +1,6 @@ -# sift configuration. P0 uses -listen and -deny-prefix; remaining keys are P1+. -# This file is an example: the service does not assume Nix store paths. +# sift configuration (/etc/sift/config.toml, or $SIFT_CONFIG). Every key is optional; +# commented values are examples. The service does not assume Nix store paths. +# Paid-search keys (BRAVE_API_KEY, TAVILY_API_KEY) go in /var/lib/sift/secret.env, not here. listen = "127.0.0.1:8890" @@ -8,11 +9,6 @@ listen = "127.0.0.1:8890" # Set on-box; do not commit residential addresses to git. # deny_prefixes = ["203.0.113.0/24"] -# [upstreams] -# searxng = "http://127.0.0.1:8888" -# dixie = "http://dixie.scylla-hammerhead.ts.net:11434/v1" -# honcho = "http://rift.scylla-hammerhead.ts.net:8000" - # user_agent = "sift/dev" # [self_ip] diff --git a/internal/api/pdf.go b/internal/api/pdf.go index 9804d08..7c7179f 100644 --- a/internal/api/pdf.go +++ b/internal/api/pdf.go @@ -18,9 +18,10 @@ import ( ) const ( - // DefaultPDFTTL bounds a cached PDF's life. Task 07; declared now so the test compiles. + // DefaultPDFTTL is how long a cached PDF is served before it is revalidated. DefaultPDFTTL = 24 * time.Hour - // DefaultPDFMaxQuestionChunks caps a digest question's length. Task 07; declared now. + // DefaultPDFMaxQuestionChunks is the longest document (in digest chunks) a question is + // answered over; a longer one is refused in favour of grep or pages. DefaultPDFMaxQuestionChunks = 200 ) diff --git a/skill/SKILL.md b/skill/SKILL.md index 5d96aac..4a0fe97 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -64,6 +64,9 @@ long one comes back as a map — an outline with page numbers, then the first pa `--question "…"` for a digest that cites each fact as `(p. N)`. - Scanned PDFs have no text layer: sift reports `no text layer (scanned PDF?)` — point it at an OCR source rather than trying to fix it here. +- A range returns at most 100 pages and grep at most 100 matches. On a very long document + `--question` is refused with `document too long for question (N chunks); use grep or pages` — + do that: `--grep` for the terms, then `--pages` around the hits. Never download a PDF and run `pdftotext` yourself; `sift` is the sanctioned extractor. @@ -92,6 +95,9 @@ Never download a PDF and run `pdftotext` yourself; `sift` is the sanctioned extr | `upstream 403/429 (likely bot-blocked)`, `hint: recover` | Site blocks non-browser clients | `sift fetch URL --recover` (Wayback → archive.today, server-side). Cite snapshots **with their date**. | | `hint: browser` | Needs JS rendering or a real browser | Use the browser tool. | | `415 unsupported type` | a content type sift cannot fetch or clean | Browser tool, or ask. | +| `503 pdf workers busy` | every PDF extraction slot is taken | Wait the `Retry-After` (10 s) and retry once. | +| `429 busy` | sift is saturated, or you already have too many fetches to that host (2) or a PDF extraction in flight | Wait for them to finish, then retry; don't fire many at once. | +| `encrypted PDF`, `unreadable PDF`, `pdf extraction timed out` | sift can't get text from this PDF | Look for an HTML or arXiv abstract version; don't retry the same URL. | | connection refused / timeout to sift itself | sift is down | Fallback below, and tell Kyle sift is down. | ## Fallback — only when sift itself is unreachable