package api
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.wntrmute.dev/kyle/sift/internal/auth"
"git.wntrmute.dev/kyle/sift/internal/digest"
"git.wntrmute.dev/kyle/sift/internal/fetch"
"git.wntrmute.dev/kyle/sift/internal/guard"
)
func setupAPI(t *testing.T, g *guard.Guard) (*Server, string) {
t.Helper()
dir := t.TempDir()
keyOut := filepath.Join(dir, "k")
keys := filepath.Join(dir, "keys.toml")
if err := auth.Add(keys, "test", "hermes-fleet", keyOut); err != nil {
t.Fatal(err)
}
kb, err := os.ReadFile(keyOut)
if err != nil {
t.Fatal(err)
}
store, err := auth.Load(keys)
if err != nil {
t.Fatal(err)
}
if g == nil {
g = &guard.Guard{}
}
f := &fetch.Fetcher{Guard: g, UA: "sift-test"}
s := New("test", g, f, store, slog.New(slog.NewTextHandler(io.Discard, nil)))
return s, strings.TrimSpace(string(kb))
}
func postJSON(t *testing.T, h http.Handler, path, key string, body any) *httptest.ResponseRecorder {
t.Helper()
b, _ := json.Marshal(body)
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b))
req.Header.Set("Content-Type", "application/json")
if key != "" {
req.Header.Set("Authorization", "Bearer "+key)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
}
func TestAuthMissingUnknown(t *testing.T) {
s, key := setupAPI(t, nil)
rec := postJSON(t, s, "/v2/scrape", "", map[string]string{"url": "https://example.com/"})
if rec.Code != 401 {
t.Fatalf("missing key: %d %s", rec.Code, rec.Body.String())
}
rec = postJSON(t, s, "/v2/scrape", "nope", map[string]string{"url": "https://example.com/"})
if rec.Code != 401 {
t.Fatalf("bad key: %d", rec.Code)
}
rec = postJSON(t, s, "/healthz", "", nil)
_ = key
}
func TestHealthzNoAuth(t *testing.T) {
s, _ := setupAPI(t, nil)
req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
rec := httptest.NewRecorder()
s.ServeHTTP(rec, req)
if rec.Code != 200 {
t.Fatal(rec.Code)
}
}
func TestDenyPrefixThroughScrape(t *testing.T) {
g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("8.8.8.8/32")}}
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://8.8.8.8/"})
if rec.Code != 403 {
t.Fatalf("got %d %s", rec.Code, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "blocked:") {
t.Fatal(rec.Body.String())
}
if strings.Contains(rec.Body.String(), `"success": true`) {
t.Fatal("success true")
}
}
func TestGuardLoopbackAndTailnet(t *testing.T) {
s, key := setupAPI(t, &guard.Guard{})
for _, u := range []string{
"http://127.0.0.1/",
"http://100.95.252.120:8000",
} {
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": u})
if rec.Code != 403 {
t.Fatalf("%s: %d %s", u, rec.Code, rec.Body.String())
}
}
}
func TestRedirectToPrivate(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "http://10.0.0.5/secret", http.StatusFound)
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL})
if rec.Code != 403 {
t.Fatalf("got %d %s", rec.Code, rec.Body.String())
}
}
func TestSkipTLSVerificationIgnored(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write([]byte("hi"))
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v2/scrape", key, map[string]any{
"url": ts.URL,
"skipTlsVerification": true,
})
if rec.Code == 200 {
t.Fatalf("bad cert succeeded: %s", rec.Body.String())
}
}
func TestDigestFallbackWhenUnset(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("
hello
"))
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "mode": "digest", "question": "what"})
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "fallback") {
t.Fatalf("want fallback: %s", rec.Body.String())
}
}
func TestCapDigestFallback(t *testing.T) {
big := strings.Repeat("word ", 50000)
out, warn := capDigestFallback(big, nil)
if digest.EstTokens(out) > digest.SmallDoc {
t.Fatalf("tokens %d", digest.EstTokens(out))
}
joined := strings.Join(warn, " ")
if !strings.Contains(joined, "digest budget") {
t.Fatalf("warn %v", warn)
}
}
func TestNever502(t *testing.T) {
s, key := setupAPI(t, nil)
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})
if rec.Code == 502 {
t.Fatal("returned 502")
}
_ = key
}
func TestCleanHTMLThroughScrape(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(`HelloHi
World
`))
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v2/scrape", key, map[string]any{
"url": ts.URL,
"formats": []string{"markdown"},
})
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
var env map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &env)
if env["success"] != true {
t.Fatal(env)
}
}
func TestJSONJQ(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"a":1,"b":2}`))
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": ".a", "mode": "clean"})
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "1") {
t.Fatal(rec.Body.String())
}
}
func TestUnsupportedType(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/pdf")
_, _ = w.Write([]byte("%PDF"))
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL})
if rec.Code != 415 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
}
func TestFetcherUsesSameGuard(t *testing.T) {
g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/32")}}
s, key := setupAPI(t, g)
if s.Fetch.Guard != g {
t.Fatal("fetcher not using the same Guard")
}
_ = key
_ = tls.VersionTLS13
}
func TestJQTooLarge(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`0`))
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": "range(20000)", "mode": "clean"})
if rec.Code != 400 || !strings.Contains(rec.Body.String(), "jq output too large") {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
}
func TestHTTPServerWriteTimeout(t *testing.T) {
s := New("test", &guard.Guard{}, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
hs := s.HTTPServer("127.0.0.1:0")
if hs.WriteTimeout != 120*time.Second {
t.Fatalf("WriteTimeout %s", hs.WriteTimeout)
}
}
type recHandler struct {
mu sync.Mutex
recs []slog.Record
}
func (h *recHandler) Enabled(context.Context, slog.Level) bool { return true }
func (h *recHandler) Handle(_ context.Context, r slog.Record) error {
h.mu.Lock()
h.recs = append(h.recs, r.Clone())
h.mu.Unlock()
return nil
}
func (h *recHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
func (h *recHandler) WithGroup(string) slog.Handler { return h }
func recHost(r slog.Record) string {
var host string
r.Attrs(func(a slog.Attr) bool {
if a.Key == "host" {
host = a.Value.String()
}
return true
})
return host
}
func recMsg(r slog.Record) string { return r.Message }
func recDump(r slog.Record) string {
var b strings.Builder
b.WriteString(r.Message)
r.Attrs(func(a slog.Attr) bool {
b.WriteByte(' ')
b.WriteString(a.Key)
b.WriteByte('=')
b.WriteString(a.Value.String())
return true
})
return b.String()
}
func TestRestrictedCallerLogsBlankHost(t *testing.T) {
dir := t.TempDir()
keys := filepath.Join(dir, "keys.toml")
fleetOut := filepath.Join(dir, "fleet")
gsOut := filepath.Join(dir, "gs")
if err := auth.Add(keys, "test", "hermes-fleet", fleetOut); err != nil {
t.Fatal(err)
}
if err := auth.Add(keys, "straylight-gs", "guardian-star", gsOut); err != nil {
t.Fatal(err)
}
fleetKey, err := os.ReadFile(fleetOut)
if err != nil {
t.Fatal(err)
}
gsKey, err := os.ReadFile(gsOut)
if err != nil {
t.Fatal(err)
}
store, err := auth.Load(keys)
if err != nil {
t.Fatal(err)
}
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
h := &recHandler{}
s := New("test", g, &fetch.Fetcher{Guard: g, UA: "sift-test"}, store, slog.New(h))
s.PermitRestricted = []string{"straylight-gs"}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(`Hiok
`))
}))
defer ts.Close()
rec := postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": ts.URL, "mode": "clean"})
if rec.Code != 200 {
t.Fatalf("gs fetch: %d %s", rec.Code, rec.Body.String())
}
var env map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
t.Fatal(err)
}
src, _ := env["source"].(map[string]any)
final, _ := src["final_url"].(string)
if !strings.Contains(final, hostOf(ts.URL)) {
t.Fatalf("final_url missing from body, got %q", final)
}
rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": "http://10.0.0.5/", "mode": "clean"})
if rec.Code != 403 {
t.Fatalf("gs refused: %d %s", rec.Code, rec.Body.String())
}
const badHost = "this-host-does-not-exist-sift-p25.invalid"
rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": "http://" + badHost + "/", "mode": "clean"})
if rec.Code != 403 {
t.Fatalf("gs nxdomain: %d %s", rec.Code, rec.Body.String())
}
rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(fleetKey)), map[string]any{"url": ts.URL, "mode": "clean"})
if rec.Code != 200 {
t.Fatalf("fleet fetch: %d %s", rec.Code, rec.Body.String())
}
var gsFetch, gsRefused, fleetFetch string
h.mu.Lock()
for _, r := range h.recs {
switch recMsg(r) {
case "fetch":
if attrCaller(r) == "straylight-gs" {
gsFetch = recHost(r)
}
if attrCaller(r) == "test" {
fleetFetch = recHost(r)
}
case "refused":
if attrCaller(r) == "straylight-gs" {
gsRefused = recHost(r)
}
}
}
h.mu.Unlock()
if gsFetch != "-" {
t.Fatalf("gs fetch host=%q want -", gsFetch)
}
if gsRefused != "-" {
t.Fatalf("gs refused host=%q want -", gsRefused)
}
wantHost := hostOf(ts.URL)
if fleetFetch != wantHost {
t.Fatalf("fleet fetch host=%q want %q", fleetFetch, wantHost)
}
h.mu.Lock()
for _, r := range h.recs {
if attrCaller(r) != "straylight-gs" {
continue
}
dump := recDump(r)
if strings.Contains(dump, badHost) || strings.Contains(dump, "10.0.0.5") {
t.Fatalf("restricted log leaked host: %s", dump)
}
}
h.mu.Unlock()
}
func attrCaller(r slog.Record) string {
var name string
r.Attrs(func(a slog.Attr) bool {
if a.Key == "caller" {
name = a.Value.String()
}
return true
})
return name
}