From e36da19f57b2e5d0374d1c1fa8aecf6156fa58fe Mon Sep 17 00:00:00 2001 From: Kyle Isom Date: Wed, 23 Sep 2026 08:33:58 -0700 Subject: [PATCH] p2.4: restricted workspace for permitted callers guardian-star context is allowed only when the authenticated caller is in permit_restricted and mapped to that workspace. Fleet callers remapped to it still refuse. CLI client.toml: SIFT_CLIENT, then $HERMES_HOME/sift, then ~/.config. --- cmd/sift/fetchcmd.go | 23 +++++++++-- cmd/sift/fetchcmd_test.go | 40 +++++++++++++++++++- cmd/sift/serve.go | 2 +- dist/sift.toml.example | 1 + docs/P2.md | 15 +++++--- docs/PLAN.md | 9 +++-- internal/api/api.go | 2 +- internal/config/config.go | 5 ++- internal/fleetctx/fleetctx.go | 43 +++++++++++++-------- internal/fleetctx/fleetctx_test.go | 61 +++++++++++++++++++++++++----- 10 files changed, 159 insertions(+), 42 deletions(-) diff --git a/cmd/sift/fetchcmd.go b/cmd/sift/fetchcmd.go index 47d56b0..cd6df37 100644 --- a/cmd/sift/fetchcmd.go +++ b/cmd/sift/fetchcmd.go @@ -19,12 +19,29 @@ type clientCfg struct { KeyFile string `toml:"key_file"` } +func clientConfigPath() string { + if p := os.Getenv("SIFT_CLIENT"); p != "" { + return p + } + if h := os.Getenv("HERMES_HOME"); h != "" { + p := filepath.Join(h, "sift", "client.toml") + if _, err := os.Stat(p); err == nil { + return p + } + } + home, err := os.UserHomeDir() + if err != nil { + return filepath.Join(".config", "sift", "client.toml") + } + return filepath.Join(home, ".config", "sift", "client.toml") +} + func loadClient() (base, key string, err error) { home, err := os.UserHomeDir() if err != nil { return "", "", err } - path := filepath.Join(home, ".config", "sift", "client.toml") + path := clientConfigPath() b, err := os.ReadFile(path) if err != nil { return "", "", fmt.Errorf("read %s: %w", path, err) @@ -132,8 +149,8 @@ func fetchMain() { } } if ctx, ok := env["context"].([]any); ok && len(ctx) > 0 { - fmt.Println("--- fleet context ---") - fmt.Println("(prior fleet knowledge, not page content)") + fmt.Println("--- prior context ---") + fmt.Println("(prior knowledge, not page content)") for _, it := range ctx { m, _ := it.(map[string]any) fmt.Printf("- %v: %v\n", m["peer"], m["content"]) diff --git a/cmd/sift/fetchcmd_test.go b/cmd/sift/fetchcmd_test.go index d3485e8..b256d2e 100644 --- a/cmd/sift/fetchcmd_test.go +++ b/cmd/sift/fetchcmd_test.go @@ -1,6 +1,11 @@ package main -import "testing" +import ( + "os" + "path/filepath" + "strings" + "testing" +) func TestParseFetchFlagsAfterURL(t *testing.T) { url, mode, _, q, noCtx, rec, _, err := parseFetchArgs([]string{"https://example.com/x", "--question", "what", "--mode", "digest", "--recover"}) @@ -28,3 +33,36 @@ func TestParseFetchRejectsExtraPositional(t *testing.T) { t.Fatal("want error") } } + +func TestClientConfigPathSIFTClient(t *testing.T) { + t.Setenv("SIFT_CLIENT", "/tmp/sift-client.toml") + if got := clientConfigPath(); got != "/tmp/sift-client.toml" { + t.Fatalf("got %q", got) + } +} + +func TestClientConfigPathHermesHome(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "sift") + if err := os.Mkdir(p, 0o700); err != nil { + t.Fatal(err) + } + cfg := filepath.Join(p, "client.toml") + if err := os.WriteFile(cfg, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("SIFT_CLIENT", "") + t.Setenv("HERMES_HOME", dir) + if got := clientConfigPath(); got != cfg { + t.Fatalf("got %q want %q", got, cfg) + } +} + +func TestClientConfigPathHermesHomeMissingFallsThrough(t *testing.T) { + t.Setenv("SIFT_CLIENT", "") + t.Setenv("HERMES_HOME", t.TempDir()) + got := clientConfigPath() + if !strings.HasSuffix(got, filepath.Join(".config", "sift", "client.toml")) { + t.Fatalf("got %q", got) + } +} diff --git a/cmd/sift/serve.go b/cmd/sift/serve.go index 6ae956b..fb70778 100644 --- a/cmd/sift/serve.go +++ b/cmd/sift/serve.go @@ -121,7 +121,7 @@ func serveMain() { if len(allow) == 0 { allow = []string{"hermes-fleet"} } - srv.Ctx = &fleetctx.Client{Base: cfg.Context.Honcho, Allow: allow, Log: log} + srv.Ctx = &fleetctx.Client{Base: cfg.Context.Honcho, Allow: allow, Permit: cfg.Context.PermitRestricted, Log: log} } if len(cfg.SelfIP.Sources) > 0 { disc := &selfip.Discoverer{ diff --git a/dist/sift.toml.example b/dist/sift.toml.example index 95edb1d..c91f03a 100644 --- a/dist/sift.toml.example +++ b/dist/sift.toml.example @@ -36,3 +36,4 @@ listen = "127.0.0.1:8890" # [context] # honcho = "http://rift.scylla-hammerhead.ts.net:8000" # workspaces = ["hermes-fleet"] +# permit_restricted = [] # caller names allowed to query guardian-star diff --git a/docs/P2.md b/docs/P2.md index c2c58e9..659e9ac 100644 --- a/docs/P2.md +++ b/docs/P2.md @@ -46,8 +46,8 @@ Out: `search` (P3), URL→content fetch cache (P4), knowledge graph. (Digest + c 3. Digest latency p50 ≤ 20 s with Dixie otherwise idle. 4. **Guardian unaffected**: during a burst of 10 concurrent digests, a guardian-shaped probe (`max_tokens` 16, thinking off) has p90 ≤ 5 s. (This is the criterion that matters most.) -5. `context` never reaches `guardian-star`: unit test + a live request with a caller mapped - to it is refused. +5. Restricted workspace: fleet caller mapped to `guardian-star` is refused; only + `permit_restricted` callers whose mapped workspace is that name get context. 6. `--recover` returns a snapshot (with `snapshot_at`) for a bot-walled URL that has a Wayback copy. @@ -101,9 +101,11 @@ invalidates; context not stored; eviction under size cap; corrupt DB → warn an list of messages with `content, created_at, peer_id, session_id, workspace_id, token_count` (verified 2026-09-22). Timeout 3 s; failure → `warnings`, empty `context`, never an error. - `ws` = the caller's workspace from `keys.toml`. **Config allowlist of workspaces** - (`context.workspaces = ["hermes-fleet"]`); anything else → no context + WARN. **Hard-coded - refusal of `guardian-star`** in code (not just config) with a test — that workspace is - compartmentalised and must never be read by sift. + (`context.workspaces`); anything else → no context + WARN. **Restricted workspace + `guardian-star`**: allowed only when the authenticated caller is in + `context.permit_restricted` *and* that caller's mapped workspace is `guardian-star`. + A fleet caller remapped to it is still refused (tested). Workspace is never taken + from the request. - Query: the question if given, else title + top 3 headings. - Return trimmed items: `{peer, content (≤300 chars), created_at, session}`; label clearly as prior fleet knowledge, not page content. @@ -156,7 +158,8 @@ digest_ms fallback`. Never log page content, questions longer than 80 chars, or falls back immediately; a waiter past `queue_wait` falls back), context cancel propagates (fake server observes the disconnect). - Chunk selection: document order restored, token budget respected, no-question path. -- Context: allowlist, `guardian-star` refused even if configured, 3 s timeout → warning. +- Context: allowlist, restricted workspace refused for unpermitted callers even if + allowlisted, permitted caller succeeds, 3 s timeout → warning. - Recover: Wayback JSON parsing (available/unavailable), `id_` rewrite, fake-success fixtures (meta-refresh to original, "Just a moment", tiny body), archive.today rotation on 429, redirect-to-private inside an archive response still blocked (guard). diff --git a/docs/PLAN.md b/docs/PLAN.md index 9398bad..179c7f2 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -35,7 +35,7 @@ attached**, using local models, so that: | Brave API key | orion `/var/lib/searx/secret.env` (`BRAVE_API_KEY`, 0640 root:searx) | verified working on **LLM Context** `GET /res/v1/llm/context?q=…` → 200, `grounding` + `sources` | | Local LLM | Dixie `http://dixie.scylla-hammerhead.ts.net:11434/v1`, model `ornith-1.5-9b-uncensored` | Qwen3.5-family thinking model, 8 slots / 96k shared KV. Per-request `chat_template_kwargs: {enable_thinking: false}` works. Shared with Honcho + three agents' guardians — budget calls. | | Embedder | same router, model `honcho-embed` (Qwen3-Embedding-0.6B, 1024-dim) | ≤2048 tokens per input on Dixie; chunk below that | -| Honcho | rift `http://rift.scylla-hammerhead.ts.net:8000`, workspace `hermes-fleet` | `POST /v3/workspaces/{ws}/search {query, filters?, limit}` = embedding/hybrid retrieval, no LLM call. **Never** query `guardian-star` from sift. | +| Honcho | rift `http://rift.scylla-hammerhead.ts.net:8000`, workspace `hermes-fleet` | `POST /v3/workspaces/{ws}/search {query, filters?, limit}` = embedding/hybrid retrieval, no LLM call. Restricted workspace `guardian-star` only for a caller in `permit_restricted` whose mapped workspace is that name. | | Hermes Firecrawl provider | `plugins/web/firecrawl/provider.py` | self-hosted via `FIRECRAWL_API_URL` (+ optional `FIRECRAWL_API_KEY`) — see §4.1 | Fleet Hermes versions differ (talos 0.19.0 uses the Firecrawl SDK; straylight 0.21.x posts raw @@ -220,8 +220,9 @@ other callers; sift must enforce independently. measured quality/latency. Timeout 45 s → fall back to `clean`. 9. **Context** (if `context`): derive 3–6 key terms (from the digest or title+headings), call Honcho `POST /v3/workspaces//search {query, limit: 5}`; return trimmed - hits. Caller → workspace mapping is static config; default `hermes-fleet`; - **`guardian-star` is never reachable from sift** unless Kyle explicitly configures it. + hits. Caller → workspace mapping is static config; default `hermes-fleet`. + Restricted workspace `guardian-star` only if the caller is in `permit_restricted` + and mapped to it. 10. **Cache**: key = normalised final URL; store clean content + hash + headers-derived TTL (default 24 h, max 7 d; `no-store` honoured). Serve cache on hit, mark `cached: true`. 11. **Log** one line per request (caller, host of URL, status, bytes, tokens, guard verdict) — @@ -272,7 +273,7 @@ tests + one live `web_extract`). |---|---|---| | **P0** skeleton | repo, flake, NixOS module, Makefile (+ `dist/sift.service`), `/healthz`, URL guard + exhaustive unit tests (table of private/tailnet/rebinding/redirect cases incl. `http://100.95.252.120:8000`, `http://dixie:11434`, `http://[fd7a:115c:a1e0::1]`, decimal/octal IPs, redirect-to-private) | tests green; Claude security review of the guard passes | | **P1** clean fetch | `/v2/scrape` (html/json/text → clean), limits, auth, logging; `sift` CLI client (`fetch --mode clean`, `--jq`) + `skill/SKILL.md` installed on hosts; deployed on orion; talos wired | `web_extract` works on talos; contract tests pass for 0.19 and 0.21; guard blocks verified live from talos | -| **P2** digest + context | `/v1/fetch` modes, Dixie digest (thinking on vs off measured), Honcho search context, token metrics | on 50 real fetches: median `returned/raw` ≤ 0.2 with spot-checked quality; p90 latency ≤ 30 s with digest; no guardian-star access possible | +| **P2** digest + context | `/v1/fetch` modes, Dixie digest (thinking on vs off measured), Honcho search context, token metrics | on 50 real fetches: median `returned/raw` ≤ 0.2 with spot-checked quality; p90 latency ≤ 30 s with digest; restricted workspace only for permit_restricted callers | | **P3** search | `/v2/search` + `/v1/search`: SearXNG + Brave LLM Context, dedupe, honcho-embed rerank | search quality spot-check vs SearXNG alone; cost per query known | | **P4** cache + provenance | SQLite cache with TTL; optional provenance (if Kyle approves) | cache hit rate reported; repeat fetches served locally | diff --git a/internal/api/api.go b/internal/api/api.go index 2c741e9..ae3bc9f 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -330,7 +330,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) { if wantCtx && s.Ctx != nil { ct0 := time.Now() query := fleetctx.Query(q, cl.Title, headings(cl.Markdown)) - items, cw := s.Ctx.Search(r.Context(), c.Workspace, query) + items, cw := s.Ctx.Search(r.Context(), c.Workspace, c.Name, query) ctxMS = time.Since(ct0).Milliseconds() if cw != "" { warn = append(warn, cw) diff --git a/internal/config/config.go b/internal/config/config.go index 60bed4d..59009f4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -34,8 +34,9 @@ type Digest struct { } type Context struct { - Honcho string `toml:"honcho"` - Workspaces []string `toml:"workspaces"` + Honcho string `toml:"honcho"` + Workspaces []string `toml:"workspaces"` + PermitRestricted []string `toml:"permit_restricted"` } // SelfIP is the [self_ip] block. diff --git a/internal/fleetctx/fleetctx.go b/internal/fleetctx/fleetctx.go index d71e124..5585136 100644 --- a/internal/fleetctx/fleetctx.go +++ b/internal/fleetctx/fleetctx.go @@ -1,4 +1,7 @@ -// Package fleetctx is Honcho retrieval for sift. Read-only. Never guardian-star. +// Package fleetctx is Honcho retrieval for sift. Read-only. +// The restricted workspace (guardian-star) is reachable only when the +// authenticated caller is in Client.Permit and the caller's mapped workspace +// is that name. Workspace is never taken from the request. package fleetctx import ( @@ -17,10 +20,10 @@ import ( ) const ( - Forbidden = "guardian-star" - Timeout = 3 * time.Second - Limit = 5 - MaxChars = 300 + Restricted = "guardian-star" + Timeout = 3 * time.Second + Limit = 5 + MaxChars = 300 ) type Item struct { @@ -33,6 +36,7 @@ type Item struct { type Client struct { Base string Allow []string + Permit []string // caller names allowed to query Restricted HTTP *http.Client Log *slog.Logger Calls atomic.Uint64 @@ -40,8 +44,20 @@ type Client struct { Failures atomic.Uint64 } -func refused(ws string) bool { - return strings.EqualFold(strings.TrimSpace(ws), Forbidden) +func restricted(ws string) bool { + return strings.EqualFold(strings.TrimSpace(ws), Restricted) +} + +func permitted(caller string, permit []string) bool { + if caller == "" { + return false + } + for _, p := range permit { + if p == caller { + return true + } + } + return false } var workspaceName = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) @@ -51,9 +67,6 @@ func workspaceOK(ws string) bool { } func allowed(ws string, allow []string) bool { - if refused(ws) { - return false - } for _, a := range allow { if a == ws { return true @@ -62,15 +75,15 @@ func allowed(ws string, allow []string) bool { return false } -// Search returns prior fleet knowledge. Failure → (nil, warning). Never an error -// that fails the fetch. guardian-star is refused in code, not only config. -func (c *Client) Search(ctx context.Context, workspace, query string) (items []Item, warn string) { +// Search returns prior knowledge for workspace. Failure → (nil, warning). Never +// an error that fails the fetch. Restricted workspace requires caller ∈ Permit. +func (c *Client) Search(ctx context.Context, workspace, caller, query string) (items []Item, warn string) { if c == nil || query == "" { return nil, "" } - if refused(workspace) { + if restricted(workspace) && !permitted(caller, c.Permit) { if c.Log != nil { - c.Log.Warn("context refused", "workspace", workspace) + c.Log.Warn("context refused", "workspace", workspace, "caller", caller) } c.Failures.Add(1) return nil, "context refused: workspace" diff --git a/internal/fleetctx/fleetctx_test.go b/internal/fleetctx/fleetctx_test.go index 3875828..3274160 100644 --- a/internal/fleetctx/fleetctx_test.go +++ b/internal/fleetctx/fleetctx_test.go @@ -11,9 +11,9 @@ import ( "time" ) -func TestGuardianStarRefusedEvenIfAllowlisted(t *testing.T) { - c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star", "hermes-fleet"}} - items, warn := c.Search(context.Background(), "guardian-star", "anything") +func TestRestrictedRefusedEvenIfAllowlisted(t *testing.T) { + c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star", "hermes-fleet"}, Permit: []string{"straylight-gs"}} + items, warn := c.Search(context.Background(), "guardian-star", "talos", "anything") if len(items) != 0 { t.Fatalf("items %v", items) } @@ -22,9 +22,52 @@ func TestGuardianStarRefusedEvenIfAllowlisted(t *testing.T) { } } +func TestRestrictedRefusedWhenFleetCallerRemapped(t *testing.T) { + c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star"}, Permit: []string{"straylight-gs"}} + items, warn := c.Search(context.Background(), "guardian-star", "straylight", "q") + if len(items) != 0 || warn == "" { + t.Fatalf("items %v warn %q", items, warn) + } +} + +func TestRestrictedAllowedForPermittedCaller(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v3/workspaces/guardian-star/search" { + t.Errorf("path %s", r.URL.Path) + } + _ = json.NewEncoder(w).Encode([]map[string]any{{ + "content": "gs already knows Y", + "created_at": "2026-09-23T00:00:00Z", + "peer_id": "kyle", + "session_id": "s1", + "workspace_id": "guardian-star", + "token_count": 4, + }}) + })) + defer ts.Close() + c := &Client{ + Base: ts.URL, + Allow: []string{"guardian-star", "hermes-fleet"}, + Permit: []string{"straylight-gs"}, + HTTP: ts.Client(), + } + items, warn := c.Search(context.Background(), "guardian-star", "straylight-gs", "Y") + if warn != "" || len(items) != 1 || items[0].Peer != "kyle" || items[0].Content != "gs already knows Y" { + t.Fatalf("%v %q", items, warn) + } +} + +func TestRestrictedSkippedIfNotAllowlisted(t *testing.T) { + c := &Client{Base: "http://127.0.0.1:1", Permit: []string{"straylight-gs"}} + _, warn := c.Search(context.Background(), "guardian-star", "straylight-gs", "q") + if warn == "" { + t.Fatal("want skip") + } +} + func TestAllowlist(t *testing.T) { c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"hermes-fleet"}} - _, warn := c.Search(context.Background(), "other-ws", "q") + _, warn := c.Search(context.Background(), "other-ws", "straylight", "q") if warn == "" { t.Fatal("want skip") } @@ -39,7 +82,7 @@ func TestTimeoutWarning(t *testing.T) { defer ts.Close() c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()} start := time.Now() - _, warn := c.Search(context.Background(), "hermes-fleet", "q") + _, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "q") if time.Since(start) > 3500*time.Millisecond { t.Fatalf("took %s", time.Since(start)) } @@ -67,7 +110,7 @@ func TestHappyPath(t *testing.T) { })) defer ts.Close() c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()} - items, warn := c.Search(context.Background(), "hermes-fleet", "X") + items, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "X") if warn != "" || len(items) != 1 || items[0].Peer != "kyle" { t.Fatalf("%v %q", items, warn) } @@ -75,7 +118,7 @@ func TestHappyPath(t *testing.T) { func TestTraversalRejected(t *testing.T) { c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"hermes-fleet"}} - _, warn := c.Search(context.Background(), "hermes-fleet/../guardian-star", "q") + _, warn := c.Search(context.Background(), "hermes-fleet/../guardian-star", "straylight-gs", "q") if warn == "" { t.Fatal("want skip") } @@ -90,7 +133,7 @@ func TestDropsForeignWorkspace(t *testing.T) { })) defer ts.Close() c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()} - items, warn := c.Search(context.Background(), "hermes-fleet", "q") + items, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "q") if warn != "" || len(items) != 1 || items[0].Content != "ok" { t.Fatalf("%v %q", items, warn) } @@ -109,7 +152,7 @@ func TestConcurrentSearch(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - _, _ = c.Search(context.Background(), "hermes-fleet", "q") + _, _ = c.Search(context.Background(), "hermes-fleet", "straylight", "q") }() } wg.Wait()