diff --git a/cmd/sift/serve.go b/cmd/sift/serve.go index fb70778..35eaf0d 100644 --- a/cmd/sift/serve.go +++ b/cmd/sift/serve.go @@ -96,6 +96,7 @@ func serveMain() { defer stop() srv := api.New(version, g, fetcher, store, log) + srv.PermitRestricted = cfg.Context.PermitRestricted if cfg.Dixie.Chat != "" { cachePath := cfg.Digest.Cache if cachePath == "" { diff --git a/docs/P2.md b/docs/P2.md index 659e9ac..3d33be3 100644 --- a/docs/P2.md +++ b/docs/P2.md @@ -149,6 +149,8 @@ CLI: `sift fetch --question Q` (implies digest + context), `--mode clean|r context calls/hits/failures; recover attempts/successes by route; token ratio histogram. Logs: one INFO line per request with `caller host mode q_len raw_tokens returned_tokens digest_ms fallback`. Never log page content, questions longer than 80 chars, or keys. +Callers in `context.permit_restricted` log `host=-` on scrape, fetch, refused, fetch-failed, +and internal lines (P2.5). `final_url` stays in the response body. ## 7. Tests diff --git a/internal/api/api.go b/internal/api/api.go index ae3bc9f..f6c2214 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -42,6 +42,9 @@ type Server struct { SelfIP *selfip.Discoverer Digest *digest.Engine Ctx *fleetctx.Client + // PermitRestricted is context.permit_restricted: those callers never + // have a fetch host written to the log (host=-). Independent of Honcho. + PermitRestricted []string global chan struct{} perMu sync.Mutex @@ -206,7 +209,7 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) { } rawTok := len(res.Body) / 4 retTok := len(md) / 4 - s.Log.Info("scrape", "caller", c.Name, "host", hostOf(res.FinalURL), "status", res.Status, "bytes", len(res.Body)) + s.Log.Info("scrape", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "status", res.Status, "bytes", len(res.Body)) writeJSON(w, 200, map[string]any{ "success": true, "data": map[string]any{ @@ -348,7 +351,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) { fb = s } } - s.Log.Info("fetch", "caller", c.Name, "host", hostOf(res.FinalURL), "mode", mode, + s.Log.Info("fetch", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "mode", mode, "q_len", qLen, "raw_tokens", len(res.Body)/4, "returned_tokens", len(content)/4, "digest_ms", digestMS, "fallback", fb) @@ -487,7 +490,7 @@ func (s *Server) writeFetchErr(w http.ResponseWriter, c *auth.Caller, rawURL str if c != nil { name = c.Name } - host := hostOf(rawURL) + host := s.logHost(c, rawURL) var fe *fetch.Error if errors.As(err, &fe) { if fe.Status == 403 { @@ -568,6 +571,26 @@ func hostOf(raw string) string { return u.Hostname() } +func permittedCaller(name string, permit []string) bool { + if name == "" { + return false + } + for _, p := range permit { + if p == name { + return true + } + } + return false +} + +// logHost is the host field for request logs. permit_restricted callers get "-". +func (s *Server) logHost(c *auth.Caller, raw string) string { + if c != nil && permittedCaller(c.Name, s.PermitRestricted) { + return "-" + } + return hostOf(raw) +} + func writeErr(w http.ResponseWriter, status int, msg, hint string) { if status == 502 { status = 500 diff --git a/internal/api/api_test.go b/internal/api/api_test.go index a0d6b73..b8beda0 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -2,6 +2,7 @@ package api import ( "bytes" + "context" "crypto/tls" "encoding/json" "io" @@ -12,6 +13,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -262,3 +264,137 @@ func TestHTTPServerWriteTimeout(t *testing.T) { t.Fatalf("WriteTimeout %s", hs.WriteTimeout) } } + +type recHandler struct { + mu sync.Mutex + recs []slog.Record +} + +func (h *recHandler) Enabled(context.Context, slog.Level) bool { return true } +func (h *recHandler) Handle(_ context.Context, r slog.Record) error { + h.mu.Lock() + h.recs = append(h.recs, r.Clone()) + h.mu.Unlock() + return nil +} +func (h *recHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *recHandler) WithGroup(string) slog.Handler { return h } + +func recHost(r slog.Record) string { + var host string + r.Attrs(func(a slog.Attr) bool { + if a.Key == "host" { + host = fmtString(a.Value) + } + return true + }) + return host +} + +func recMsg(r slog.Record) string { return r.Message } + +func fmtString(v slog.Value) string { + if v.Kind() == slog.KindString { + return v.String() + } + return v.Any().(string) +} + +func TestRestrictedCallerLogsBlankHost(t *testing.T) { + dir := t.TempDir() + keys := filepath.Join(dir, "keys.toml") + fleetOut := filepath.Join(dir, "fleet") + gsOut := filepath.Join(dir, "gs") + if err := auth.Add(keys, "test", "hermes-fleet", fleetOut); err != nil { + t.Fatal(err) + } + if err := auth.Add(keys, "straylight-gs", "guardian-star", gsOut); err != nil { + t.Fatal(err) + } + fleetKey, err := os.ReadFile(fleetOut) + if err != nil { + t.Fatal(err) + } + gsKey, err := os.ReadFile(gsOut) + if err != nil { + t.Fatal(err) + } + store, err := auth.Load(keys) + if err != nil { + t.Fatal(err) + } + g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}} + h := &recHandler{} + s := New("test", g, &fetch.Fetcher{Guard: g, UA: "sift-test"}, store, slog.New(h)) + s.PermitRestricted = []string{"straylight-gs"} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(`Hi

ok

`)) + })) + defer ts.Close() + + rec := postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": ts.URL, "mode": "clean"}) + if rec.Code != 200 { + t.Fatalf("gs fetch: %d %s", rec.Code, rec.Body.String()) + } + var env map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil { + t.Fatal(err) + } + src, _ := env["source"].(map[string]any) + final, _ := src["final_url"].(string) + if !strings.Contains(final, hostOf(ts.URL)) { + t.Fatalf("final_url still in body, got %q", final) + } + + rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": "http://10.0.0.5/", "mode": "clean"}) + if rec.Code != 403 { + t.Fatalf("gs refused: %d %s", rec.Code, rec.Body.String()) + } + + rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(fleetKey)), map[string]any{"url": ts.URL, "mode": "clean"}) + if rec.Code != 200 { + t.Fatalf("fleet fetch: %d %s", rec.Code, rec.Body.String()) + } + + var gsFetch, gsRefused, fleetFetch string + h.mu.Lock() + for _, r := range h.recs { + switch recMsg(r) { + case "fetch": + if attrCaller(r) == "straylight-gs" { + gsFetch = recHost(r) + } + if attrCaller(r) == "test" { + fleetFetch = recHost(r) + } + case "refused": + if attrCaller(r) == "straylight-gs" { + gsRefused = recHost(r) + } + } + } + h.mu.Unlock() + if gsFetch != "-" { + t.Fatalf("gs fetch host=%q want -", gsFetch) + } + if gsRefused != "-" { + t.Fatalf("gs refused host=%q want -", gsRefused) + } + wantHost := hostOf(ts.URL) + if fleetFetch != wantHost { + t.Fatalf("fleet fetch host=%q want %q", fleetFetch, wantHost) + } +} + +func attrCaller(r slog.Record) string { + var name string + r.Attrs(func(a slog.Attr) bool { + if a.Key == "caller" { + name = a.Value.String() + } + return true + }) + return name +}