package api import ( "bytes" "crypto/tls" "encoding/json" "io" "log/slog" "net/http" "net/http/httptest" "net/netip" "os" "path/filepath" "strings" "testing" "git.wntrmute.dev/kyle/sift/internal/auth" "git.wntrmute.dev/kyle/sift/internal/fetch" "git.wntrmute.dev/kyle/sift/internal/guard" ) func setupAPI(t *testing.T, g *guard.Guard) (*Server, string) { t.Helper() dir := t.TempDir() keyOut := filepath.Join(dir, "k") keys := filepath.Join(dir, "keys.toml") if err := auth.Add(keys, "test", "hermes-fleet", keyOut); err != nil { t.Fatal(err) } kb, err := os.ReadFile(keyOut) if err != nil { t.Fatal(err) } store, err := auth.Load(keys) if err != nil { t.Fatal(err) } if g == nil { g = &guard.Guard{} } f := &fetch.Fetcher{Guard: g, UA: "sift-test"} s := New("test", g, f, store, slog.New(slog.NewTextHandler(io.Discard, nil))) return s, strings.TrimSpace(string(kb)) } func postJSON(t *testing.T, h http.Handler, path, key string, body any) *httptest.ResponseRecorder { t.Helper() b, _ := json.Marshal(body) req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b)) req.Header.Set("Content-Type", "application/json") if key != "" { req.Header.Set("Authorization", "Bearer "+key) } rec := httptest.NewRecorder() h.ServeHTTP(rec, req) return rec } func TestAuthMissingUnknown(t *testing.T) { s, key := setupAPI(t, nil) rec := postJSON(t, s, "/v2/scrape", "", map[string]string{"url": "https://example.com/"}) if rec.Code != 401 { t.Fatalf("missing key: %d %s", rec.Code, rec.Body.String()) } rec = postJSON(t, s, "/v2/scrape", "nope", map[string]string{"url": "https://example.com/"}) if rec.Code != 401 { t.Fatalf("bad key: %d", rec.Code) } rec = postJSON(t, s, "/healthz", "", nil) _ = key } func TestHealthzNoAuth(t *testing.T) { s, _ := setupAPI(t, nil) req := httptest.NewRequest(http.MethodGet, "/healthz", nil) rec := httptest.NewRecorder() s.ServeHTTP(rec, req) if rec.Code != 200 { t.Fatal(rec.Code) } } func TestDenyPrefixThroughScrape(t *testing.T) { g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("8.8.8.8/32")}} s, key := setupAPI(t, g) rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://8.8.8.8/"}) if rec.Code != 403 { t.Fatalf("got %d %s", rec.Code, rec.Body.String()) } if !strings.Contains(rec.Body.String(), "blocked:") { t.Fatal(rec.Body.String()) } if strings.Contains(rec.Body.String(), `"success": true`) { t.Fatal("success true") } } func TestGuardLoopbackAndTailnet(t *testing.T) { s, key := setupAPI(t, &guard.Guard{}) for _, u := range []string{ "http://127.0.0.1/", "http://100.95.252.120:8000", } { rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": u}) if rec.Code != 403 { t.Fatalf("%s: %d %s", u, rec.Code, rec.Body.String()) } } } func TestRedirectToPrivate(t *testing.T) { g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "http://10.0.0.5/secret", http.StatusFound) })) defer ts.Close() s, key := setupAPI(t, g) rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL}) if rec.Code != 403 { t.Fatalf("got %d %s", rec.Code, rec.Body.String()) } } func TestSkipTLSVerificationIgnored(t *testing.T) { g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}} ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") _, _ = w.Write([]byte("hi")) })) defer ts.Close() s, key := setupAPI(t, g) rec := postJSON(t, s, "/v2/scrape", key, map[string]any{ "url": ts.URL, "skipTlsVerification": true, }) if rec.Code == 200 { t.Fatalf("bad cert succeeded: %s", rec.Body.String()) } } func TestDigestFallbackWhenUnset(t *testing.T) { g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") _, _ = w.Write([]byte("

hello

")) })) defer ts.Close() s, key := setupAPI(t, g) rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "mode": "digest", "question": "what"}) if rec.Code != 200 { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } if !strings.Contains(rec.Body.String(), "fallback") { t.Fatalf("want fallback: %s", rec.Body.String()) } } func TestNever502(t *testing.T) { s, key := setupAPI(t, nil) rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"}) if rec.Code == 502 { t.Fatal("returned 502") } _ = key } func TestCleanHTMLThroughScrape(t *testing.T) { g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html; charset=utf-8") _, _ = w.Write([]byte(`Hello

Hi

World

`)) })) defer ts.Close() s, key := setupAPI(t, g) rec := postJSON(t, s, "/v2/scrape", key, map[string]any{ "url": ts.URL, "formats": []string{"markdown"}, }) if rec.Code != 200 { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } var env map[string]any _ = json.Unmarshal(rec.Body.Bytes(), &env) if env["success"] != true { t.Fatal(env) } } func TestJSONJQ(t *testing.T) { g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"a":1,"b":2}`)) })) defer ts.Close() s, key := setupAPI(t, g) rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": ".a", "mode": "clean"}) if rec.Code != 200 { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } if !strings.Contains(rec.Body.String(), "1") { t.Fatal(rec.Body.String()) } } func TestUnsupportedType(t *testing.T) { g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/pdf") _, _ = w.Write([]byte("%PDF")) })) defer ts.Close() s, key := setupAPI(t, g) rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL}) if rec.Code != 415 { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } } func TestFetcherUsesSameGuard(t *testing.T) { g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/32")}} s, key := setupAPI(t, g) if s.Fetch.Guard != g { t.Fatal("fetcher not using the same Guard") } _ = key _ = tls.VersionTLS13 } func TestJQTooLarge(t *testing.T) { g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`0`)) })) defer ts.Close() s, key := setupAPI(t, g) rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": "range(20000)", "mode": "clean"}) if rec.Code != 400 || !strings.Contains(rec.Body.String(), "jq output too large") { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } }