p2.4: restricted workspace for permitted callers

guardian-star context is allowed only when the authenticated caller is in
permit_restricted and mapped to that workspace. Fleet callers remapped to it
still refuse. CLI client.toml: SIFT_CLIENT, then $HERMES_HOME/sift, then ~/.config.
This commit is contained in:
2026-09-23 08:33:58 -07:00
parent 14eb7808ce
commit e36da19f57
10 changed files with 159 additions and 42 deletions
+20 -3
View File
@@ -19,12 +19,29 @@ type clientCfg struct {
KeyFile string `toml:"key_file"`
}
func clientConfigPath() string {
if p := os.Getenv("SIFT_CLIENT"); p != "" {
return p
}
if h := os.Getenv("HERMES_HOME"); h != "" {
p := filepath.Join(h, "sift", "client.toml")
if _, err := os.Stat(p); err == nil {
return p
}
}
home, err := os.UserHomeDir()
if err != nil {
return filepath.Join(".config", "sift", "client.toml")
}
return filepath.Join(home, ".config", "sift", "client.toml")
}
func loadClient() (base, key string, err error) {
home, err := os.UserHomeDir()
if err != nil {
return "", "", err
}
path := filepath.Join(home, ".config", "sift", "client.toml")
path := clientConfigPath()
b, err := os.ReadFile(path)
if err != nil {
return "", "", fmt.Errorf("read %s: %w", path, err)
@@ -132,8 +149,8 @@ func fetchMain() {
}
}
if ctx, ok := env["context"].([]any); ok && len(ctx) > 0 {
fmt.Println("--- fleet context ---")
fmt.Println("(prior fleet knowledge, not page content)")
fmt.Println("--- prior context ---")
fmt.Println("(prior knowledge, not page content)")
for _, it := range ctx {
m, _ := it.(map[string]any)
fmt.Printf("- %v: %v\n", m["peer"], m["content"])
+39 -1
View File
@@ -1,6 +1,11 @@
package main
import "testing"
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestParseFetchFlagsAfterURL(t *testing.T) {
url, mode, _, q, noCtx, rec, _, err := parseFetchArgs([]string{"https://example.com/x", "--question", "what", "--mode", "digest", "--recover"})
@@ -28,3 +33,36 @@ func TestParseFetchRejectsExtraPositional(t *testing.T) {
t.Fatal("want error")
}
}
func TestClientConfigPathSIFTClient(t *testing.T) {
t.Setenv("SIFT_CLIENT", "/tmp/sift-client.toml")
if got := clientConfigPath(); got != "/tmp/sift-client.toml" {
t.Fatalf("got %q", got)
}
}
func TestClientConfigPathHermesHome(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "sift")
if err := os.Mkdir(p, 0o700); err != nil {
t.Fatal(err)
}
cfg := filepath.Join(p, "client.toml")
if err := os.WriteFile(cfg, []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("SIFT_CLIENT", "")
t.Setenv("HERMES_HOME", dir)
if got := clientConfigPath(); got != cfg {
t.Fatalf("got %q want %q", got, cfg)
}
}
func TestClientConfigPathHermesHomeMissingFallsThrough(t *testing.T) {
t.Setenv("SIFT_CLIENT", "")
t.Setenv("HERMES_HOME", t.TempDir())
got := clientConfigPath()
if !strings.HasSuffix(got, filepath.Join(".config", "sift", "client.toml")) {
t.Fatalf("got %q", got)
}
}
+1 -1
View File
@@ -121,7 +121,7 @@ func serveMain() {
if len(allow) == 0 {
allow = []string{"hermes-fleet"}
}
srv.Ctx = &fleetctx.Client{Base: cfg.Context.Honcho, Allow: allow, Log: log}
srv.Ctx = &fleetctx.Client{Base: cfg.Context.Honcho, Allow: allow, Permit: cfg.Context.PermitRestricted, Log: log}
}
if len(cfg.SelfIP.Sources) > 0 {
disc := &selfip.Discoverer{
+1
View File
@@ -36,3 +36,4 @@ listen = "127.0.0.1:8890"
# [context]
# honcho = "http://rift.scylla-hammerhead.ts.net:8000"
# workspaces = ["hermes-fleet"]
# permit_restricted = [] # caller names allowed to query guardian-star
+9 -6
View File
@@ -46,8 +46,8 @@ Out: `search` (P3), URL→content fetch cache (P4), knowledge graph. (Digest + c
3. Digest latency p50 ≤ 20 s with Dixie otherwise idle.
4. **Guardian unaffected**: during a burst of 10 concurrent digests, a guardian-shaped probe
(`max_tokens` 16, thinking off) has p90 ≤ 5 s. (This is the criterion that matters most.)
5. `context` never reaches `guardian-star`: unit test + a live request with a caller mapped
to it is refused.
5. Restricted workspace: fleet caller mapped to `guardian-star` is refused; only
`permit_restricted` callers whose mapped workspace is that name get context.
6. `--recover` returns a snapshot (with `snapshot_at`) for a bot-walled URL that has a Wayback
copy.
@@ -101,9 +101,11 @@ invalidates; context not stored; eviction under size cap; corrupt DB → warn an
list of messages with `content, created_at, peer_id, session_id, workspace_id, token_count`
(verified 2026-09-22). Timeout 3 s; failure → `warnings`, empty `context`, never an error.
- `ws` = the caller's workspace from `keys.toml`. **Config allowlist of workspaces**
(`context.workspaces = ["hermes-fleet"]`); anything else → no context + WARN. **Hard-coded
refusal of `guardian-star`** in code (not just config) with a test — that workspace is
compartmentalised and must never be read by sift.
(`context.workspaces`); anything else → no context + WARN. **Restricted workspace
`guardian-star`**: allowed only when the authenticated caller is in
`context.permit_restricted` *and* that caller's mapped workspace is `guardian-star`.
A fleet caller remapped to it is still refused (tested). Workspace is never taken
from the request.
- Query: the question if given, else title + top 3 headings.
- Return trimmed items: `{peer, content (≤300 chars), created_at, session}`; label clearly
as prior fleet knowledge, not page content.
@@ -156,7 +158,8 @@ digest_ms fallback`. Never log page content, questions longer than 80 chars, or
falls back immediately; a waiter past `queue_wait` falls back), context cancel propagates
(fake server observes the disconnect).
- Chunk selection: document order restored, token budget respected, no-question path.
- Context: allowlist, `guardian-star` refused even if configured, 3 s timeout → warning.
- Context: allowlist, restricted workspace refused for unpermitted callers even if
allowlisted, permitted caller succeeds, 3 s timeout → warning.
- Recover: Wayback JSON parsing (available/unavailable), `id_` rewrite, fake-success fixtures
(meta-refresh to original, "Just a moment", tiny body), archive.today rotation on 429,
redirect-to-private inside an archive response still blocked (guard).
+5 -4
View File
@@ -35,7 +35,7 @@ attached**, using local models, so that:
| Brave API key | orion `/var/lib/searx/secret.env` (`BRAVE_API_KEY`, 0640 root:searx) | verified working on **LLM Context** `GET /res/v1/llm/context?q=…` → 200, `grounding` + `sources` |
| Local LLM | Dixie `http://dixie.scylla-hammerhead.ts.net:11434/v1`, model `ornith-1.5-9b-uncensored` | Qwen3.5-family thinking model, 8 slots / 96k shared KV. Per-request `chat_template_kwargs: {enable_thinking: false}` works. Shared with Honcho + three agents' guardians — budget calls. |
| Embedder | same router, model `honcho-embed` (Qwen3-Embedding-0.6B, 1024-dim) | ≤2048 tokens per input on Dixie; chunk below that |
| Honcho | rift `http://rift.scylla-hammerhead.ts.net:8000`, workspace `hermes-fleet` | `POST /v3/workspaces/{ws}/search {query, filters?, limit}` = embedding/hybrid retrieval, no LLM call. **Never** query `guardian-star` from sift. |
| Honcho | rift `http://rift.scylla-hammerhead.ts.net:8000`, workspace `hermes-fleet` | `POST /v3/workspaces/{ws}/search {query, filters?, limit}` = embedding/hybrid retrieval, no LLM call. Restricted workspace `guardian-star` only for a caller in `permit_restricted` whose mapped workspace is that name. |
| Hermes Firecrawl provider | `plugins/web/firecrawl/provider.py` | self-hosted via `FIRECRAWL_API_URL` (+ optional `FIRECRAWL_API_KEY`) — see §4.1 |
Fleet Hermes versions differ (talos 0.19.0 uses the Firecrawl SDK; straylight 0.21.x posts raw
@@ -220,8 +220,9 @@ other callers; sift must enforce independently.
measured quality/latency. Timeout 45 s → fall back to `clean`.
9. **Context** (if `context`): derive 3–6 key terms (from the digest or title+headings), call
Honcho `POST /v3/workspaces/<caller-workspace>/search {query, limit: 5}`; return trimmed
hits. Caller → workspace mapping is static config; default `hermes-fleet`;
**`guardian-star` is never reachable from sift** unless Kyle explicitly configures it.
hits. Caller → workspace mapping is static config; default `hermes-fleet`.
Restricted workspace `guardian-star` only if the caller is in `permit_restricted`
and mapped to it.
10. **Cache**: key = normalised final URL; store clean content + hash + headers-derived TTL
(default 24 h, max 7 d; `no-store` honoured). Serve cache on hit, mark `cached: true`.
11. **Log** one line per request (caller, host of URL, status, bytes, tokens, guard verdict) —
@@ -272,7 +273,7 @@ tests + one live `web_extract`).
|---|---|---|
| **P0** skeleton | repo, flake, NixOS module, Makefile (+ `dist/sift.service`), `/healthz`, URL guard + exhaustive unit tests (table of private/tailnet/rebinding/redirect cases incl. `http://100.95.252.120:8000`, `http://dixie:11434`, `http://[fd7a:115c:a1e0::1]`, decimal/octal IPs, redirect-to-private) | tests green; Claude security review of the guard passes |
| **P1** clean fetch | `/v2/scrape` (html/json/text → clean), limits, auth, logging; `sift` CLI client (`fetch --mode clean`, `--jq`) + `skill/SKILL.md` installed on hosts; deployed on orion; talos wired | `web_extract` works on talos; contract tests pass for 0.19 and 0.21; guard blocks verified live from talos |
| **P2** digest + context | `/v1/fetch` modes, Dixie digest (thinking on vs off measured), Honcho search context, token metrics | on 50 real fetches: median `returned/raw` ≤ 0.2 with spot-checked quality; p90 latency ≤ 30 s with digest; no guardian-star access possible |
| **P2** digest + context | `/v1/fetch` modes, Dixie digest (thinking on vs off measured), Honcho search context, token metrics | on 50 real fetches: median `returned/raw` ≤ 0.2 with spot-checked quality; p90 latency ≤ 30 s with digest; restricted workspace only for permit_restricted callers |
| **P3** search | `/v2/search` + `/v1/search`: SearXNG + Brave LLM Context, dedupe, honcho-embed rerank | search quality spot-check vs SearXNG alone; cost per query known |
| **P4** cache + provenance | SQLite cache with TTL; optional provenance (if Kyle approves) | cache hit rate reported; repeat fetches served locally |
+1 -1
View File
@@ -330,7 +330,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if wantCtx && s.Ctx != nil {
ct0 := time.Now()
query := fleetctx.Query(q, cl.Title, headings(cl.Markdown))
items, cw := s.Ctx.Search(r.Context(), c.Workspace, query)
items, cw := s.Ctx.Search(r.Context(), c.Workspace, c.Name, query)
ctxMS = time.Since(ct0).Milliseconds()
if cw != "" {
warn = append(warn, cw)
+1
View File
@@ -36,6 +36,7 @@ type Digest struct {
type Context struct {
Honcho string `toml:"honcho"`
Workspaces []string `toml:"workspaces"`
PermitRestricted []string `toml:"permit_restricted"`
}
// SelfIP is the [self_ip] block.
+25 -12
View File
@@ -1,4 +1,7 @@
// Package fleetctx is Honcho retrieval for sift. Read-only. Never guardian-star.
// Package fleetctx is Honcho retrieval for sift. Read-only.
// The restricted workspace (guardian-star) is reachable only when the
// authenticated caller is in Client.Permit and the caller's mapped workspace
// is that name. Workspace is never taken from the request.
package fleetctx
import (
@@ -17,7 +20,7 @@ import (
)
const (
Forbidden = "guardian-star"
Restricted = "guardian-star"
Timeout = 3 * time.Second
Limit = 5
MaxChars = 300
@@ -33,6 +36,7 @@ type Item struct {
type Client struct {
Base string
Allow []string
Permit []string // caller names allowed to query Restricted
HTTP *http.Client
Log *slog.Logger
Calls atomic.Uint64
@@ -40,8 +44,20 @@ type Client struct {
Failures atomic.Uint64
}
func refused(ws string) bool {
return strings.EqualFold(strings.TrimSpace(ws), Forbidden)
func restricted(ws string) bool {
return strings.EqualFold(strings.TrimSpace(ws), Restricted)
}
func permitted(caller string, permit []string) bool {
if caller == "" {
return false
}
for _, p := range permit {
if p == caller {
return true
}
}
return false
}
var workspaceName = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
@@ -51,9 +67,6 @@ func workspaceOK(ws string) bool {
}
func allowed(ws string, allow []string) bool {
if refused(ws) {
return false
}
for _, a := range allow {
if a == ws {
return true
@@ -62,15 +75,15 @@ func allowed(ws string, allow []string) bool {
return false
}
// Search returns prior fleet knowledge. Failure → (nil, warning). Never an error
// that fails the fetch. guardian-star is refused in code, not only config.
func (c *Client) Search(ctx context.Context, workspace, query string) (items []Item, warn string) {
// Search returns prior knowledge for workspace. Failure → (nil, warning). Never
// an error that fails the fetch. Restricted workspace requires caller ∈ Permit.
func (c *Client) Search(ctx context.Context, workspace, caller, query string) (items []Item, warn string) {
if c == nil || query == "" {
return nil, ""
}
if refused(workspace) {
if restricted(workspace) && !permitted(caller, c.Permit) {
if c.Log != nil {
c.Log.Warn("context refused", "workspace", workspace)
c.Log.Warn("context refused", "workspace", workspace, "caller", caller)
}
c.Failures.Add(1)
return nil, "context refused: workspace"
+52 -9
View File
@@ -11,9 +11,9 @@ import (
"time"
)
func TestGuardianStarRefusedEvenIfAllowlisted(t *testing.T) {
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star", "hermes-fleet"}}
items, warn := c.Search(context.Background(), "guardian-star", "anything")
func TestRestrictedRefusedEvenIfAllowlisted(t *testing.T) {
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star", "hermes-fleet"}, Permit: []string{"straylight-gs"}}
items, warn := c.Search(context.Background(), "guardian-star", "talos", "anything")
if len(items) != 0 {
t.Fatalf("items %v", items)
}
@@ -22,9 +22,52 @@ func TestGuardianStarRefusedEvenIfAllowlisted(t *testing.T) {
}
}
func TestRestrictedRefusedWhenFleetCallerRemapped(t *testing.T) {
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star"}, Permit: []string{"straylight-gs"}}
items, warn := c.Search(context.Background(), "guardian-star", "straylight", "q")
if len(items) != 0 || warn == "" {
t.Fatalf("items %v warn %q", items, warn)
}
}
func TestRestrictedAllowedForPermittedCaller(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v3/workspaces/guardian-star/search" {
t.Errorf("path %s", r.URL.Path)
}
_ = json.NewEncoder(w).Encode([]map[string]any{{
"content": "gs already knows Y",
"created_at": "2026-09-23T00:00:00Z",
"peer_id": "kyle",
"session_id": "s1",
"workspace_id": "guardian-star",
"token_count": 4,
}})
}))
defer ts.Close()
c := &Client{
Base: ts.URL,
Allow: []string{"guardian-star", "hermes-fleet"},
Permit: []string{"straylight-gs"},
HTTP: ts.Client(),
}
items, warn := c.Search(context.Background(), "guardian-star", "straylight-gs", "Y")
if warn != "" || len(items) != 1 || items[0].Peer != "kyle" || items[0].Content != "gs already knows Y" {
t.Fatalf("%v %q", items, warn)
}
}
func TestRestrictedSkippedIfNotAllowlisted(t *testing.T) {
c := &Client{Base: "http://127.0.0.1:1", Permit: []string{"straylight-gs"}}
_, warn := c.Search(context.Background(), "guardian-star", "straylight-gs", "q")
if warn == "" {
t.Fatal("want skip")
}
}
func TestAllowlist(t *testing.T) {
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"hermes-fleet"}}
_, warn := c.Search(context.Background(), "other-ws", "q")
_, warn := c.Search(context.Background(), "other-ws", "straylight", "q")
if warn == "" {
t.Fatal("want skip")
}
@@ -39,7 +82,7 @@ func TestTimeoutWarning(t *testing.T) {
defer ts.Close()
c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()}
start := time.Now()
_, warn := c.Search(context.Background(), "hermes-fleet", "q")
_, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "q")
if time.Since(start) > 3500*time.Millisecond {
t.Fatalf("took %s", time.Since(start))
}
@@ -67,7 +110,7 @@ func TestHappyPath(t *testing.T) {
}))
defer ts.Close()
c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()}
items, warn := c.Search(context.Background(), "hermes-fleet", "X")
items, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "X")
if warn != "" || len(items) != 1 || items[0].Peer != "kyle" {
t.Fatalf("%v %q", items, warn)
}
@@ -75,7 +118,7 @@ func TestHappyPath(t *testing.T) {
func TestTraversalRejected(t *testing.T) {
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"hermes-fleet"}}
_, warn := c.Search(context.Background(), "hermes-fleet/../guardian-star", "q")
_, warn := c.Search(context.Background(), "hermes-fleet/../guardian-star", "straylight-gs", "q")
if warn == "" {
t.Fatal("want skip")
}
@@ -90,7 +133,7 @@ func TestDropsForeignWorkspace(t *testing.T) {
}))
defer ts.Close()
c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()}
items, warn := c.Search(context.Background(), "hermes-fleet", "q")
items, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "q")
if warn != "" || len(items) != 1 || items[0].Content != "ok" {
t.Fatalf("%v %q", items, warn)
}
@@ -109,7 +152,7 @@ func TestConcurrentSearch(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
_, _ = c.Search(context.Background(), "hermes-fleet", "q")
_, _ = c.Search(context.Background(), "hermes-fleet", "straylight", "q")
}()
}
wg.Wait()