sift/internal/api/api_test.go

package api

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"io"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"git.wntrmute.dev/kyle/sift/internal/auth"
	"git.wntrmute.dev/kyle/sift/internal/fetch"
	"git.wntrmute.dev/kyle/sift/internal/guard"
)

func setupAPI(t *testing.T, g *guard.Guard) (*Server, string) {
	t.Helper()
	dir := t.TempDir()
	keyOut := filepath.Join(dir, "k")
	keys := filepath.Join(dir, "keys.toml")
	if err := auth.Add(keys, "test", "hermes-fleet", keyOut); err != nil {
		t.Fatal(err)
	}
	kb, err := os.ReadFile(keyOut)
	if err != nil {
		t.Fatal(err)
	}
	store, err := auth.Load(keys)
	if err != nil {
		t.Fatal(err)
	}
	if g == nil {
		g = &guard.Guard{}
	}
	f := &fetch.Fetcher{Guard: g, UA: "sift-test"}
	s := New("test", g, f, store, slog.New(slog.NewTextHandler(io.Discard, nil)))
	return s, strings.TrimSpace(string(kb))
}

func postJSON(t *testing.T, h http.Handler, path, key string, body any) *httptest.ResponseRecorder {
	t.Helper()
	b, _ := json.Marshal(body)
	req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b))
	req.Header.Set("Content-Type", "application/json")
	if key != "" {
		req.Header.Set("Authorization", "Bearer "+key)
	}
	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, req)
	return rec
}

func TestAuthMissingUnknown(t *testing.T) {
	s, key := setupAPI(t, nil)
	rec := postJSON(t, s, "/v2/scrape", "", map[string]string{"url": "https://example.com/"})
	if rec.Code != 401 {
		t.Fatalf("missing key: %d %s", rec.Code, rec.Body.String())
	}
	rec = postJSON(t, s, "/v2/scrape", "nope", map[string]string{"url": "https://example.com/"})
	if rec.Code != 401 {
		t.Fatalf("bad key: %d", rec.Code)
	}
	rec = postJSON(t, s, "/healthz", "", nil)
	_ = key
}

func TestHealthzNoAuth(t *testing.T) {
	s, _ := setupAPI(t, nil)
	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
	rec := httptest.NewRecorder()
	s.ServeHTTP(rec, req)
	if rec.Code != 200 {
		t.Fatal(rec.Code)
	}
}

func TestDenyPrefixThroughScrape(t *testing.T) {
	g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("8.8.8.8/32")}}
	s, key := setupAPI(t, g)
	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://8.8.8.8/"})
	if rec.Code != 403 {
		t.Fatalf("got %d %s", rec.Code, rec.Body.String())
	}
	if !strings.Contains(rec.Body.String(), "blocked:") {
		t.Fatal(rec.Body.String())
	}
	if strings.Contains(rec.Body.String(), `"success": true`) {
		t.Fatal("success true")
	}
}

func TestGuardLoopbackAndTailnet(t *testing.T) {
	s, key := setupAPI(t, &guard.Guard{})
	for _, u := range []string{
		"http://127.0.0.1/",
		"http://100.95.252.120:8000",
	} {
		rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": u})
		if rec.Code != 403 {
			t.Fatalf("%s: %d %s", u, rec.Code, rec.Body.String())
		}
	}
}

func TestRedirectToPrivate(t *testing.T) {
	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "http://10.0.0.5/secret", http.StatusFound)
	}))
	defer ts.Close()
	s, key := setupAPI(t, g)
	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL})
	if rec.Code != 403 {
		t.Fatalf("got %d %s", rec.Code, rec.Body.String())
	}
}

func TestSkipTLSVerificationIgnored(t *testing.T) {
	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		_, _ = w.Write([]byte("hi"))
	}))
	defer ts.Close()
	s, key := setupAPI(t, g)
	rec := postJSON(t, s, "/v2/scrape", key, map[string]any{
		"url":                 ts.URL,
		"skipTlsVerification": true,
	})
	if rec.Code == 200 {
		t.Fatalf("bad cert succeeded: %s", rec.Body.String())
	}
}

func TestDigestFallbackWhenUnset(t *testing.T) {
	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html")
		_, _ = w.Write([]byte("<html><body><p>hello</p></body></html>"))
	}))
	defer ts.Close()
	s, key := setupAPI(t, g)
	rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "mode": "digest", "question": "what"})
	if rec.Code != 200 {
		t.Fatalf("%d %s", rec.Code, rec.Body.String())
	}
	if !strings.Contains(rec.Body.String(), "fallback") {
		t.Fatalf("want fallback: %s", rec.Body.String())
	}
}

func TestNever502(t *testing.T) {
	s, key := setupAPI(t, nil)
	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})
	if rec.Code == 502 {
		t.Fatal("returned 502")
	}
	_ = key
}

func TestCleanHTMLThroughScrape(t *testing.T) {
	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		_, _ = w.Write([]byte(`<!doctype html><html><head><title>Hello</title></head><body><article><h1>Hi</h1><p>World</p></article></body></html>`))
	}))
	defer ts.Close()
	s, key := setupAPI(t, g)
	rec := postJSON(t, s, "/v2/scrape", key, map[string]any{
		"url":     ts.URL,
		"formats": []string{"markdown"},
	})
	if rec.Code != 200 {
		t.Fatalf("%d %s", rec.Code, rec.Body.String())
	}
	var env map[string]any
	_ = json.Unmarshal(rec.Body.Bytes(), &env)
	if env["success"] != true {
		t.Fatal(env)
	}
}

func TestJSONJQ(t *testing.T) {
	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"a":1,"b":2}`))
	}))
	defer ts.Close()
	s, key := setupAPI(t, g)
	rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": ".a", "mode": "clean"})
	if rec.Code != 200 {
		t.Fatalf("%d %s", rec.Code, rec.Body.String())
	}
	if !strings.Contains(rec.Body.String(), "1") {
		t.Fatal(rec.Body.String())
	}
}

func TestUnsupportedType(t *testing.T) {
	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/pdf")
		_, _ = w.Write([]byte("%PDF"))
	}))
	defer ts.Close()
	s, key := setupAPI(t, g)
	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL})
	if rec.Code != 415 {
		t.Fatalf("%d %s", rec.Code, rec.Body.String())
	}
}

func TestFetcherUsesSameGuard(t *testing.T) {
	g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/32")}}
	s, key := setupAPI(t, g)
	if s.Fetch.Guard != g {
		t.Fatal("fetcher not using the same Guard")
	}
	_ = key
	_ = tls.VersionTLS13
}

func TestJQTooLarge(t *testing.T) {
	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`0`))
	}))
	defer ts.Close()
	s, key := setupAPI(t, g)
	rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": "range(20000)", "mode": "clean"})
	if rec.Code != 400 || !strings.Contains(rec.Body.String(), "jq output too large") {
		t.Fatalf("%d %s", rec.Code, rec.Body.String())
	}
}

func TestHTTPServerWriteTimeout(t *testing.T) {
	s := New("test", &guard.Guard{}, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
	hs := s.HTTPServer("127.0.0.1:0")
	if hs.WriteTimeout != 120*time.Second {
		t.Fatalf("WriteTimeout %s", hs.WriteTimeout)
	}
}