p2.4: restricted workspace for permitted callers
guardian-star context is allowed only when the authenticated caller is in permit_restricted and mapped to that workspace. Fleet callers remapped to it still refuse. CLI client.toml: SIFT_CLIENT, then $HERMES_HOME/sift, then ~/.config.
This commit is contained in:
+20
-3
@@ -19,12 +19,29 @@ type clientCfg struct {
|
||||
KeyFile string `toml:"key_file"`
|
||||
}
|
||||
|
||||
func clientConfigPath() string {
|
||||
if p := os.Getenv("SIFT_CLIENT"); p != "" {
|
||||
return p
|
||||
}
|
||||
if h := os.Getenv("HERMES_HOME"); h != "" {
|
||||
p := filepath.Join(h, "sift", "client.toml")
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return filepath.Join(".config", "sift", "client.toml")
|
||||
}
|
||||
return filepath.Join(home, ".config", "sift", "client.toml")
|
||||
}
|
||||
|
||||
func loadClient() (base, key string, err error) {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
path := filepath.Join(home, ".config", "sift", "client.toml")
|
||||
path := clientConfigPath()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("read %s: %w", path, err)
|
||||
@@ -132,8 +149,8 @@ func fetchMain() {
|
||||
}
|
||||
}
|
||||
if ctx, ok := env["context"].([]any); ok && len(ctx) > 0 {
|
||||
fmt.Println("--- fleet context ---")
|
||||
fmt.Println("(prior fleet knowledge, not page content)")
|
||||
fmt.Println("--- prior context ---")
|
||||
fmt.Println("(prior knowledge, not page content)")
|
||||
for _, it := range ctx {
|
||||
m, _ := it.(map[string]any)
|
||||
fmt.Printf("- %v: %v\n", m["peer"], m["content"])
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseFetchFlagsAfterURL(t *testing.T) {
|
||||
url, mode, _, q, noCtx, rec, _, err := parseFetchArgs([]string{"https://example.com/x", "--question", "what", "--mode", "digest", "--recover"})
|
||||
@@ -28,3 +33,36 @@ func TestParseFetchRejectsExtraPositional(t *testing.T) {
|
||||
t.Fatal("want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientConfigPathSIFTClient(t *testing.T) {
|
||||
t.Setenv("SIFT_CLIENT", "/tmp/sift-client.toml")
|
||||
if got := clientConfigPath(); got != "/tmp/sift-client.toml" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientConfigPathHermesHome(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "sift")
|
||||
if err := os.Mkdir(p, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := filepath.Join(p, "client.toml")
|
||||
if err := os.WriteFile(cfg, []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("SIFT_CLIENT", "")
|
||||
t.Setenv("HERMES_HOME", dir)
|
||||
if got := clientConfigPath(); got != cfg {
|
||||
t.Fatalf("got %q want %q", got, cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientConfigPathHermesHomeMissingFallsThrough(t *testing.T) {
|
||||
t.Setenv("SIFT_CLIENT", "")
|
||||
t.Setenv("HERMES_HOME", t.TempDir())
|
||||
got := clientConfigPath()
|
||||
if !strings.HasSuffix(got, filepath.Join(".config", "sift", "client.toml")) {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -121,7 +121,7 @@ func serveMain() {
|
||||
if len(allow) == 0 {
|
||||
allow = []string{"hermes-fleet"}
|
||||
}
|
||||
srv.Ctx = &fleetctx.Client{Base: cfg.Context.Honcho, Allow: allow, Log: log}
|
||||
srv.Ctx = &fleetctx.Client{Base: cfg.Context.Honcho, Allow: allow, Permit: cfg.Context.PermitRestricted, Log: log}
|
||||
}
|
||||
if len(cfg.SelfIP.Sources) > 0 {
|
||||
disc := &selfip.Discoverer{
|
||||
|
||||
Vendored
+1
@@ -36,3 +36,4 @@ listen = "127.0.0.1:8890"
|
||||
# [context]
|
||||
# honcho = "http://rift.scylla-hammerhead.ts.net:8000"
|
||||
# workspaces = ["hermes-fleet"]
|
||||
# permit_restricted = [] # caller names allowed to query guardian-star
|
||||
|
||||
+9
-6
@@ -46,8 +46,8 @@ Out: `search` (P3), URL→content fetch cache (P4), knowledge graph. (Digest + c
|
||||
3. Digest latency p50 ≤ 20 s with Dixie otherwise idle.
|
||||
4. **Guardian unaffected**: during a burst of 10 concurrent digests, a guardian-shaped probe
|
||||
(`max_tokens` 16, thinking off) has p90 ≤ 5 s. (This is the criterion that matters most.)
|
||||
5. `context` never reaches `guardian-star`: unit test + a live request with a caller mapped
|
||||
to it is refused.
|
||||
5. Restricted workspace: fleet caller mapped to `guardian-star` is refused; only
|
||||
`permit_restricted` callers whose mapped workspace is that name get context.
|
||||
6. `--recover` returns a snapshot (with `snapshot_at`) for a bot-walled URL that has a Wayback
|
||||
copy.
|
||||
|
||||
@@ -101,9 +101,11 @@ invalidates; context not stored; eviction under size cap; corrupt DB → warn an
|
||||
list of messages with `content, created_at, peer_id, session_id, workspace_id, token_count`
|
||||
(verified 2026-09-22). Timeout 3 s; failure → `warnings`, empty `context`, never an error.
|
||||
- `ws` = the caller's workspace from `keys.toml`. **Config allowlist of workspaces**
|
||||
(`context.workspaces = ["hermes-fleet"]`); anything else → no context + WARN. **Hard-coded
|
||||
refusal of `guardian-star`** in code (not just config) with a test — that workspace is
|
||||
compartmentalised and must never be read by sift.
|
||||
(`context.workspaces`); anything else → no context + WARN. **Restricted workspace
|
||||
`guardian-star`**: allowed only when the authenticated caller is in
|
||||
`context.permit_restricted` *and* that caller's mapped workspace is `guardian-star`.
|
||||
A fleet caller remapped to it is still refused (tested). Workspace is never taken
|
||||
from the request.
|
||||
- Query: the question if given, else title + top 3 headings.
|
||||
- Return trimmed items: `{peer, content (≤300 chars), created_at, session}`; label clearly
|
||||
as prior fleet knowledge, not page content.
|
||||
@@ -156,7 +158,8 @@ digest_ms fallback`. Never log page content, questions longer than 80 chars, or
|
||||
falls back immediately; a waiter past `queue_wait` falls back), context cancel propagates
|
||||
(fake server observes the disconnect).
|
||||
- Chunk selection: document order restored, token budget respected, no-question path.
|
||||
- Context: allowlist, `guardian-star` refused even if configured, 3 s timeout → warning.
|
||||
- Context: allowlist, restricted workspace refused for unpermitted callers even if
|
||||
allowlisted, permitted caller succeeds, 3 s timeout → warning.
|
||||
- Recover: Wayback JSON parsing (available/unavailable), `id_` rewrite, fake-success fixtures
|
||||
(meta-refresh to original, "Just a moment", tiny body), archive.today rotation on 429,
|
||||
redirect-to-private inside an archive response still blocked (guard).
|
||||
|
||||
+5
-4
@@ -35,7 +35,7 @@ attached**, using local models, so that:
|
||||
| Brave API key | orion `/var/lib/searx/secret.env` (`BRAVE_API_KEY`, 0640 root:searx) | verified working on **LLM Context** `GET /res/v1/llm/context?q=…` → 200, `grounding` + `sources` |
|
||||
| Local LLM | Dixie `http://dixie.scylla-hammerhead.ts.net:11434/v1`, model `ornith-1.5-9b-uncensored` | Qwen3.5-family thinking model, 8 slots / 96k shared KV. Per-request `chat_template_kwargs: {enable_thinking: false}` works. Shared with Honcho + three agents' guardians — budget calls. |
|
||||
| Embedder | same router, model `honcho-embed` (Qwen3-Embedding-0.6B, 1024-dim) | ≤2048 tokens per input on Dixie; chunk below that |
|
||||
| Honcho | rift `http://rift.scylla-hammerhead.ts.net:8000`, workspace `hermes-fleet` | `POST /v3/workspaces/{ws}/search {query, filters?, limit}` = embedding/hybrid retrieval, no LLM call. **Never** query `guardian-star` from sift. |
|
||||
| Honcho | rift `http://rift.scylla-hammerhead.ts.net:8000`, workspace `hermes-fleet` | `POST /v3/workspaces/{ws}/search {query, filters?, limit}` = embedding/hybrid retrieval, no LLM call. Restricted workspace `guardian-star` only for a caller in `permit_restricted` whose mapped workspace is that name. |
|
||||
| Hermes Firecrawl provider | `plugins/web/firecrawl/provider.py` | self-hosted via `FIRECRAWL_API_URL` (+ optional `FIRECRAWL_API_KEY`) — see §4.1 |
|
||||
|
||||
Fleet Hermes versions differ (talos 0.19.0 uses the Firecrawl SDK; straylight 0.21.x posts raw
|
||||
@@ -220,8 +220,9 @@ other callers; sift must enforce independently.
|
||||
measured quality/latency. Timeout 45 s → fall back to `clean`.
|
||||
9. **Context** (if `context`): derive 3–6 key terms (from the digest or title+headings), call
|
||||
Honcho `POST /v3/workspaces/<caller-workspace>/search {query, limit: 5}`; return trimmed
|
||||
hits. Caller → workspace mapping is static config; default `hermes-fleet`;
|
||||
**`guardian-star` is never reachable from sift** unless Kyle explicitly configures it.
|
||||
hits. Caller → workspace mapping is static config; default `hermes-fleet`.
|
||||
Restricted workspace `guardian-star` only if the caller is in `permit_restricted`
|
||||
and mapped to it.
|
||||
10. **Cache**: key = normalised final URL; store clean content + hash + headers-derived TTL
|
||||
(default 24 h, max 7 d; `no-store` honoured). Serve cache on hit, mark `cached: true`.
|
||||
11. **Log** one line per request (caller, host of URL, status, bytes, tokens, guard verdict) —
|
||||
@@ -272,7 +273,7 @@ tests + one live `web_extract`).
|
||||
|---|---|---|
|
||||
| **P0** skeleton | repo, flake, NixOS module, Makefile (+ `dist/sift.service`), `/healthz`, URL guard + exhaustive unit tests (table of private/tailnet/rebinding/redirect cases incl. `http://100.95.252.120:8000`, `http://dixie:11434`, `http://[fd7a:115c:a1e0::1]`, decimal/octal IPs, redirect-to-private) | tests green; Claude security review of the guard passes |
|
||||
| **P1** clean fetch | `/v2/scrape` (html/json/text → clean), limits, auth, logging; `sift` CLI client (`fetch --mode clean`, `--jq`) + `skill/SKILL.md` installed on hosts; deployed on orion; talos wired | `web_extract` works on talos; contract tests pass for 0.19 and 0.21; guard blocks verified live from talos |
|
||||
| **P2** digest + context | `/v1/fetch` modes, Dixie digest (thinking on vs off measured), Honcho search context, token metrics | on 50 real fetches: median `returned/raw` ≤ 0.2 with spot-checked quality; p90 latency ≤ 30 s with digest; no guardian-star access possible |
|
||||
| **P2** digest + context | `/v1/fetch` modes, Dixie digest (thinking on vs off measured), Honcho search context, token metrics | on 50 real fetches: median `returned/raw` ≤ 0.2 with spot-checked quality; p90 latency ≤ 30 s with digest; restricted workspace only for permit_restricted callers |
|
||||
| **P3** search | `/v2/search` + `/v1/search`: SearXNG + Brave LLM Context, dedupe, honcho-embed rerank | search quality spot-check vs SearXNG alone; cost per query known |
|
||||
| **P4** cache + provenance | SQLite cache with TTL; optional provenance (if Kyle approves) | cache hit rate reported; repeat fetches served locally |
|
||||
|
||||
|
||||
+1
-1
@@ -330,7 +330,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
|
||||
if wantCtx && s.Ctx != nil {
|
||||
ct0 := time.Now()
|
||||
query := fleetctx.Query(q, cl.Title, headings(cl.Markdown))
|
||||
items, cw := s.Ctx.Search(r.Context(), c.Workspace, query)
|
||||
items, cw := s.Ctx.Search(r.Context(), c.Workspace, c.Name, query)
|
||||
ctxMS = time.Since(ct0).Milliseconds()
|
||||
if cw != "" {
|
||||
warn = append(warn, cw)
|
||||
|
||||
@@ -36,6 +36,7 @@ type Digest struct {
|
||||
type Context struct {
|
||||
Honcho string `toml:"honcho"`
|
||||
Workspaces []string `toml:"workspaces"`
|
||||
PermitRestricted []string `toml:"permit_restricted"`
|
||||
}
|
||||
|
||||
// SelfIP is the [self_ip] block.
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
// Package fleetctx is Honcho retrieval for sift. Read-only. Never guardian-star.
|
||||
// Package fleetctx is Honcho retrieval for sift. Read-only.
|
||||
// The restricted workspace (guardian-star) is reachable only when the
|
||||
// authenticated caller is in Client.Permit and the caller's mapped workspace
|
||||
// is that name. Workspace is never taken from the request.
|
||||
package fleetctx
|
||||
|
||||
import (
|
||||
@@ -17,7 +20,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
Forbidden = "guardian-star"
|
||||
Restricted = "guardian-star"
|
||||
Timeout = 3 * time.Second
|
||||
Limit = 5
|
||||
MaxChars = 300
|
||||
@@ -33,6 +36,7 @@ type Item struct {
|
||||
type Client struct {
|
||||
Base string
|
||||
Allow []string
|
||||
Permit []string // caller names allowed to query Restricted
|
||||
HTTP *http.Client
|
||||
Log *slog.Logger
|
||||
Calls atomic.Uint64
|
||||
@@ -40,8 +44,20 @@ type Client struct {
|
||||
Failures atomic.Uint64
|
||||
}
|
||||
|
||||
func refused(ws string) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(ws), Forbidden)
|
||||
func restricted(ws string) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(ws), Restricted)
|
||||
}
|
||||
|
||||
func permitted(caller string, permit []string) bool {
|
||||
if caller == "" {
|
||||
return false
|
||||
}
|
||||
for _, p := range permit {
|
||||
if p == caller {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var workspaceName = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
@@ -51,9 +67,6 @@ func workspaceOK(ws string) bool {
|
||||
}
|
||||
|
||||
func allowed(ws string, allow []string) bool {
|
||||
if refused(ws) {
|
||||
return false
|
||||
}
|
||||
for _, a := range allow {
|
||||
if a == ws {
|
||||
return true
|
||||
@@ -62,15 +75,15 @@ func allowed(ws string, allow []string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Search returns prior fleet knowledge. Failure → (nil, warning). Never an error
|
||||
// that fails the fetch. guardian-star is refused in code, not only config.
|
||||
func (c *Client) Search(ctx context.Context, workspace, query string) (items []Item, warn string) {
|
||||
// Search returns prior knowledge for workspace. Failure → (nil, warning). Never
|
||||
// an error that fails the fetch. Restricted workspace requires caller ∈ Permit.
|
||||
func (c *Client) Search(ctx context.Context, workspace, caller, query string) (items []Item, warn string) {
|
||||
if c == nil || query == "" {
|
||||
return nil, ""
|
||||
}
|
||||
if refused(workspace) {
|
||||
if restricted(workspace) && !permitted(caller, c.Permit) {
|
||||
if c.Log != nil {
|
||||
c.Log.Warn("context refused", "workspace", workspace)
|
||||
c.Log.Warn("context refused", "workspace", workspace, "caller", caller)
|
||||
}
|
||||
c.Failures.Add(1)
|
||||
return nil, "context refused: workspace"
|
||||
|
||||
@@ -11,9 +11,9 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGuardianStarRefusedEvenIfAllowlisted(t *testing.T) {
|
||||
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star", "hermes-fleet"}}
|
||||
items, warn := c.Search(context.Background(), "guardian-star", "anything")
|
||||
func TestRestrictedRefusedEvenIfAllowlisted(t *testing.T) {
|
||||
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star", "hermes-fleet"}, Permit: []string{"straylight-gs"}}
|
||||
items, warn := c.Search(context.Background(), "guardian-star", "talos", "anything")
|
||||
if len(items) != 0 {
|
||||
t.Fatalf("items %v", items)
|
||||
}
|
||||
@@ -22,9 +22,52 @@ func TestGuardianStarRefusedEvenIfAllowlisted(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedRefusedWhenFleetCallerRemapped(t *testing.T) {
|
||||
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"guardian-star"}, Permit: []string{"straylight-gs"}}
|
||||
items, warn := c.Search(context.Background(), "guardian-star", "straylight", "q")
|
||||
if len(items) != 0 || warn == "" {
|
||||
t.Fatalf("items %v warn %q", items, warn)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedAllowedForPermittedCaller(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v3/workspaces/guardian-star/search" {
|
||||
t.Errorf("path %s", r.URL.Path)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode([]map[string]any{{
|
||||
"content": "gs already knows Y",
|
||||
"created_at": "2026-09-23T00:00:00Z",
|
||||
"peer_id": "kyle",
|
||||
"session_id": "s1",
|
||||
"workspace_id": "guardian-star",
|
||||
"token_count": 4,
|
||||
}})
|
||||
}))
|
||||
defer ts.Close()
|
||||
c := &Client{
|
||||
Base: ts.URL,
|
||||
Allow: []string{"guardian-star", "hermes-fleet"},
|
||||
Permit: []string{"straylight-gs"},
|
||||
HTTP: ts.Client(),
|
||||
}
|
||||
items, warn := c.Search(context.Background(), "guardian-star", "straylight-gs", "Y")
|
||||
if warn != "" || len(items) != 1 || items[0].Peer != "kyle" || items[0].Content != "gs already knows Y" {
|
||||
t.Fatalf("%v %q", items, warn)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedSkippedIfNotAllowlisted(t *testing.T) {
|
||||
c := &Client{Base: "http://127.0.0.1:1", Permit: []string{"straylight-gs"}}
|
||||
_, warn := c.Search(context.Background(), "guardian-star", "straylight-gs", "q")
|
||||
if warn == "" {
|
||||
t.Fatal("want skip")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowlist(t *testing.T) {
|
||||
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"hermes-fleet"}}
|
||||
_, warn := c.Search(context.Background(), "other-ws", "q")
|
||||
_, warn := c.Search(context.Background(), "other-ws", "straylight", "q")
|
||||
if warn == "" {
|
||||
t.Fatal("want skip")
|
||||
}
|
||||
@@ -39,7 +82,7 @@ func TestTimeoutWarning(t *testing.T) {
|
||||
defer ts.Close()
|
||||
c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()}
|
||||
start := time.Now()
|
||||
_, warn := c.Search(context.Background(), "hermes-fleet", "q")
|
||||
_, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "q")
|
||||
if time.Since(start) > 3500*time.Millisecond {
|
||||
t.Fatalf("took %s", time.Since(start))
|
||||
}
|
||||
@@ -67,7 +110,7 @@ func TestHappyPath(t *testing.T) {
|
||||
}))
|
||||
defer ts.Close()
|
||||
c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()}
|
||||
items, warn := c.Search(context.Background(), "hermes-fleet", "X")
|
||||
items, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "X")
|
||||
if warn != "" || len(items) != 1 || items[0].Peer != "kyle" {
|
||||
t.Fatalf("%v %q", items, warn)
|
||||
}
|
||||
@@ -75,7 +118,7 @@ func TestHappyPath(t *testing.T) {
|
||||
|
||||
func TestTraversalRejected(t *testing.T) {
|
||||
c := &Client{Base: "http://127.0.0.1:1", Allow: []string{"hermes-fleet"}}
|
||||
_, warn := c.Search(context.Background(), "hermes-fleet/../guardian-star", "q")
|
||||
_, warn := c.Search(context.Background(), "hermes-fleet/../guardian-star", "straylight-gs", "q")
|
||||
if warn == "" {
|
||||
t.Fatal("want skip")
|
||||
}
|
||||
@@ -90,7 +133,7 @@ func TestDropsForeignWorkspace(t *testing.T) {
|
||||
}))
|
||||
defer ts.Close()
|
||||
c := &Client{Base: ts.URL, Allow: []string{"hermes-fleet"}, HTTP: ts.Client()}
|
||||
items, warn := c.Search(context.Background(), "hermes-fleet", "q")
|
||||
items, warn := c.Search(context.Background(), "hermes-fleet", "straylight", "q")
|
||||
if warn != "" || len(items) != 1 || items[0].Content != "ok" {
|
||||
t.Fatalf("%v %q", items, warn)
|
||||
}
|
||||
@@ -109,7 +152,7 @@ func TestConcurrentSearch(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _ = c.Search(context.Background(), "hermes-fleet", "q")
|
||||
_, _ = c.Search(context.Background(), "hermes-fleet", "straylight", "q")
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
Reference in New Issue
Block a user