p2.5: log host=- for permit_restricted callers

Scrape, fetch, refused, fetch-failed, and internal lines redact the host for callers in context.permit_restricted. final_url stays in the response. Test captures slog records, not the journal.
2026-09-23 11:40:45 -07:00
parent e36da19f57
commit 85113f9c0b
4 changed files with 165 additions and 3 deletions
@@ -96,6 +96,7 @@ func serveMain() {
 	defer stop()

 	srv := api.New(version, g, fetcher, store, log)
+	srv.PermitRestricted = cfg.Context.PermitRestricted
 	if cfg.Dixie.Chat != "" {
 		cachePath := cfg.Digest.Cache
 		if cachePath == "" {
@@ -149,6 +149,8 @@ CLI: `sift fetch <url> --question Q` (implies digest + context), `--mode clean|r
 context calls/hits/failures; recover attempts/successes by route; token ratio histogram.
 Logs: one INFO line per request with `caller host mode q_len raw_tokens returned_tokens
 digest_ms fallback`. Never log page content, questions longer than 80 chars, or keys.
+Callers in `context.permit_restricted` log `host=-` on scrape, fetch, refused, fetch-failed,
+and internal lines (P2.5). `final_url` stays in the response body.

 ## 7. Tests

@@ -42,6 +42,9 @@ type Server struct {
 	SelfIP  *selfip.Discoverer
 	Digest  *digest.Engine
 	Ctx     *fleetctx.Client
+	// PermitRestricted is context.permit_restricted: those callers never
+	// have a fetch host written to the log (host=-). Independent of Honcho.
+	PermitRestricted []string

 	global chan struct{}
 	perMu  sync.Mutex
@@ -206,7 +209,7 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
 	}
 	rawTok := len(res.Body) / 4
 	retTok := len(md) / 4
-	s.Log.Info("scrape", "caller", c.Name, "host", hostOf(res.FinalURL), "status", res.Status, "bytes", len(res.Body))
+	s.Log.Info("scrape", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "status", res.Status, "bytes", len(res.Body))
 	writeJSON(w, 200, map[string]any{
 		"success": true,
 		"data": map[string]any{
@@ -348,7 +351,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
 			fb = s
 		}
 	}
-	s.Log.Info("fetch", "caller", c.Name, "host", hostOf(res.FinalURL), "mode", mode,
+	s.Log.Info("fetch", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "mode", mode,
 		"q_len", qLen, "raw_tokens", len(res.Body)/4, "returned_tokens", len(content)/4,
 		"digest_ms", digestMS, "fallback", fb)

@@ -487,7 +490,7 @@ func (s *Server) writeFetchErr(w http.ResponseWriter, c *auth.Caller, rawURL str
 	if c != nil {
 		name = c.Name
 	}
-	host := hostOf(rawURL)
+	host := s.logHost(c, rawURL)
 	var fe *fetch.Error
 	if errors.As(err, &fe) {
 		if fe.Status == 403 {
@@ -568,6 +571,26 @@ func hostOf(raw string) string {
 	return u.Hostname()
 }

+func permittedCaller(name string, permit []string) bool {
+	if name == "" {
+		return false
+	}
+	for _, p := range permit {
+		if p == name {
+			return true
+		}
+	}
+	return false
+}
+
+// logHost is the host field for request logs. permit_restricted callers get "-".
+func (s *Server) logHost(c *auth.Caller, raw string) string {
+	if c != nil && permittedCaller(c.Name, s.PermitRestricted) {
+		return "-"
+	}
+	return hostOf(raw)
+}
+
 func writeErr(w http.ResponseWriter, status int, msg, hint string) {
 	if status == 502 {
 		status = 500
@@ -2,6 +2,7 @@ package api

 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"io"
@@ -12,6 +13,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"

@@ -262,3 +264,137 @@ func TestHTTPServerWriteTimeout(t *testing.T) {
 		t.Fatalf("WriteTimeout %s", hs.WriteTimeout)
 	}
 }
+
+type recHandler struct {
+	mu   sync.Mutex
+	recs []slog.Record
+}
+
+func (h *recHandler) Enabled(context.Context, slog.Level) bool { return true }
+func (h *recHandler) Handle(_ context.Context, r slog.Record) error {
+	h.mu.Lock()
+	h.recs = append(h.recs, r.Clone())
+	h.mu.Unlock()
+	return nil
+}
+func (h *recHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
+func (h *recHandler) WithGroup(string) slog.Handler      { return h }
+
+func recHost(r slog.Record) string {
+	var host string
+	r.Attrs(func(a slog.Attr) bool {
+		if a.Key == "host" {
+			host = fmtString(a.Value)
+		}
+		return true
+	})
+	return host
+}
+
+func recMsg(r slog.Record) string { return r.Message }
+
+func fmtString(v slog.Value) string {
+	if v.Kind() == slog.KindString {
+		return v.String()
+	}
+	return v.Any().(string)
+}
+
+func TestRestrictedCallerLogsBlankHost(t *testing.T) {
+	dir := t.TempDir()
+	keys := filepath.Join(dir, "keys.toml")
+	fleetOut := filepath.Join(dir, "fleet")
+	gsOut := filepath.Join(dir, "gs")
+	if err := auth.Add(keys, "test", "hermes-fleet", fleetOut); err != nil {
+		t.Fatal(err)
+	}
+	if err := auth.Add(keys, "straylight-gs", "guardian-star", gsOut); err != nil {
+		t.Fatal(err)
+	}
+	fleetKey, err := os.ReadFile(fleetOut)
+	if err != nil {
+		t.Fatal(err)
+	}
+	gsKey, err := os.ReadFile(gsOut)
+	if err != nil {
+		t.Fatal(err)
+	}
+	store, err := auth.Load(keys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
+	h := &recHandler{}
+	s := New("test", g, &fetch.Fetcher{Guard: g, UA: "sift-test"}, store, slog.New(h))
+	s.PermitRestricted = []string{"straylight-gs"}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(`<!doctype html><html><head><title>Hi</title></head><body><article><p>ok</p></article></body></html>`))
+	}))
+	defer ts.Close()
+
+	rec := postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": ts.URL, "mode": "clean"})
+	if rec.Code != 200 {
+		t.Fatalf("gs fetch: %d %s", rec.Code, rec.Body.String())
+	}
+	var env map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
+		t.Fatal(err)
+	}
+	src, _ := env["source"].(map[string]any)
+	final, _ := src["final_url"].(string)
+	if !strings.Contains(final, hostOf(ts.URL)) {
+		t.Fatalf("final_url still in body, got %q", final)
+	}
+
+	rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": "http://10.0.0.5/", "mode": "clean"})
+	if rec.Code != 403 {
+		t.Fatalf("gs refused: %d %s", rec.Code, rec.Body.String())
+	}
+
+	rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(fleetKey)), map[string]any{"url": ts.URL, "mode": "clean"})
+	if rec.Code != 200 {
+		t.Fatalf("fleet fetch: %d %s", rec.Code, rec.Body.String())
+	}
+
+	var gsFetch, gsRefused, fleetFetch string
+	h.mu.Lock()
+	for _, r := range h.recs {
+		switch recMsg(r) {
+		case "fetch":
+			if attrCaller(r) == "straylight-gs" {
+				gsFetch = recHost(r)
+			}
+			if attrCaller(r) == "test" {
+				fleetFetch = recHost(r)
+			}
+		case "refused":
+			if attrCaller(r) == "straylight-gs" {
+				gsRefused = recHost(r)
+			}
+		}
+	}
+	h.mu.Unlock()
+	if gsFetch != "-" {
+		t.Fatalf("gs fetch host=%q want -", gsFetch)
+	}
+	if gsRefused != "-" {
+		t.Fatalf("gs refused host=%q want -", gsRefused)
+	}
+	wantHost := hostOf(ts.URL)
+	if fleetFetch != wantHost {
+		t.Fatalf("fleet fetch host=%q want %q", fleetFetch, wantHost)
+	}
+}
+
+func attrCaller(r slog.Record) string {
+	var name string
+	r.Attrs(func(a slog.Attr) bool {
+		if a.Key == "caller" {
+			name = a.Value.String()
+		}
+		return true
+	})
+	return name
+}