p2.5: log host=- for permit_restricted callers
Scrape, fetch, refused, fetch-failed, and internal lines redact the host for callers in context.permit_restricted. final_url stays in the response. Test captures slog records, not the journal.
This commit is contained in:
+26
-3
@@ -42,6 +42,9 @@ type Server struct {
|
||||
SelfIP *selfip.Discoverer
|
||||
Digest *digest.Engine
|
||||
Ctx *fleetctx.Client
|
||||
// PermitRestricted is context.permit_restricted: those callers never
|
||||
// have a fetch host written to the log (host=-). Independent of Honcho.
|
||||
PermitRestricted []string
|
||||
|
||||
global chan struct{}
|
||||
perMu sync.Mutex
|
||||
@@ -206,7 +209,7 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
rawTok := len(res.Body) / 4
|
||||
retTok := len(md) / 4
|
||||
s.Log.Info("scrape", "caller", c.Name, "host", hostOf(res.FinalURL), "status", res.Status, "bytes", len(res.Body))
|
||||
s.Log.Info("scrape", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "status", res.Status, "bytes", len(res.Body))
|
||||
writeJSON(w, 200, map[string]any{
|
||||
"success": true,
|
||||
"data": map[string]any{
|
||||
@@ -348,7 +351,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
|
||||
fb = s
|
||||
}
|
||||
}
|
||||
s.Log.Info("fetch", "caller", c.Name, "host", hostOf(res.FinalURL), "mode", mode,
|
||||
s.Log.Info("fetch", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "mode", mode,
|
||||
"q_len", qLen, "raw_tokens", len(res.Body)/4, "returned_tokens", len(content)/4,
|
||||
"digest_ms", digestMS, "fallback", fb)
|
||||
|
||||
@@ -487,7 +490,7 @@ func (s *Server) writeFetchErr(w http.ResponseWriter, c *auth.Caller, rawURL str
|
||||
if c != nil {
|
||||
name = c.Name
|
||||
}
|
||||
host := hostOf(rawURL)
|
||||
host := s.logHost(c, rawURL)
|
||||
var fe *fetch.Error
|
||||
if errors.As(err, &fe) {
|
||||
if fe.Status == 403 {
|
||||
@@ -568,6 +571,26 @@ func hostOf(raw string) string {
|
||||
return u.Hostname()
|
||||
}
|
||||
|
||||
func permittedCaller(name string, permit []string) bool {
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
for _, p := range permit {
|
||||
if p == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// logHost is the host field for request logs. permit_restricted callers get "-".
|
||||
func (s *Server) logHost(c *auth.Caller, raw string) string {
|
||||
if c != nil && permittedCaller(c.Name, s.PermitRestricted) {
|
||||
return "-"
|
||||
}
|
||||
return hostOf(raw)
|
||||
}
|
||||
|
||||
func writeErr(w http.ResponseWriter, status int, msg, hint string) {
|
||||
if status == 502 {
|
||||
status = 500
|
||||
|
||||
@@ -2,6 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"io"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -262,3 +264,137 @@ func TestHTTPServerWriteTimeout(t *testing.T) {
|
||||
t.Fatalf("WriteTimeout %s", hs.WriteTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
type recHandler struct {
|
||||
mu sync.Mutex
|
||||
recs []slog.Record
|
||||
}
|
||||
|
||||
func (h *recHandler) Enabled(context.Context, slog.Level) bool { return true }
|
||||
func (h *recHandler) Handle(_ context.Context, r slog.Record) error {
|
||||
h.mu.Lock()
|
||||
h.recs = append(h.recs, r.Clone())
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
func (h *recHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
|
||||
func (h *recHandler) WithGroup(string) slog.Handler { return h }
|
||||
|
||||
func recHost(r slog.Record) string {
|
||||
var host string
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
if a.Key == "host" {
|
||||
host = fmtString(a.Value)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return host
|
||||
}
|
||||
|
||||
func recMsg(r slog.Record) string { return r.Message }
|
||||
|
||||
func fmtString(v slog.Value) string {
|
||||
if v.Kind() == slog.KindString {
|
||||
return v.String()
|
||||
}
|
||||
return v.Any().(string)
|
||||
}
|
||||
|
||||
func TestRestrictedCallerLogsBlankHost(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keys := filepath.Join(dir, "keys.toml")
|
||||
fleetOut := filepath.Join(dir, "fleet")
|
||||
gsOut := filepath.Join(dir, "gs")
|
||||
if err := auth.Add(keys, "test", "hermes-fleet", fleetOut); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := auth.Add(keys, "straylight-gs", "guardian-star", gsOut); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fleetKey, err := os.ReadFile(fleetOut)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gsKey, err := os.ReadFile(gsOut)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err := auth.Load(keys)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
||||
h := &recHandler{}
|
||||
s := New("test", g, &fetch.Fetcher{Guard: g, UA: "sift-test"}, store, slog.New(h))
|
||||
s.PermitRestricted = []string{"straylight-gs"}
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(`<!doctype html><html><head><title>Hi</title></head><body><article><p>ok</p></article></body></html>`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
rec := postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": ts.URL, "mode": "clean"})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("gs fetch: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var env map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, _ := env["source"].(map[string]any)
|
||||
final, _ := src["final_url"].(string)
|
||||
if !strings.Contains(final, hostOf(ts.URL)) {
|
||||
t.Fatalf("final_url still in body, got %q", final)
|
||||
}
|
||||
|
||||
rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": "http://10.0.0.5/", "mode": "clean"})
|
||||
if rec.Code != 403 {
|
||||
t.Fatalf("gs refused: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(fleetKey)), map[string]any{"url": ts.URL, "mode": "clean"})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("fleet fetch: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var gsFetch, gsRefused, fleetFetch string
|
||||
h.mu.Lock()
|
||||
for _, r := range h.recs {
|
||||
switch recMsg(r) {
|
||||
case "fetch":
|
||||
if attrCaller(r) == "straylight-gs" {
|
||||
gsFetch = recHost(r)
|
||||
}
|
||||
if attrCaller(r) == "test" {
|
||||
fleetFetch = recHost(r)
|
||||
}
|
||||
case "refused":
|
||||
if attrCaller(r) == "straylight-gs" {
|
||||
gsRefused = recHost(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if gsFetch != "-" {
|
||||
t.Fatalf("gs fetch host=%q want -", gsFetch)
|
||||
}
|
||||
if gsRefused != "-" {
|
||||
t.Fatalf("gs refused host=%q want -", gsRefused)
|
||||
}
|
||||
wantHost := hostOf(ts.URL)
|
||||
if fleetFetch != wantHost {
|
||||
t.Fatalf("fleet fetch host=%q want %q", fleetFetch, wantHost)
|
||||
}
|
||||
}
|
||||
|
||||
func attrCaller(r slog.Record) string {
|
||||
var name string
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
if a.Key == "caller" {
|
||||
name = a.Value.String()
|
||||
}
|
||||
return true
|
||||
})
|
||||
return name
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user