kyle
85113f9c0b
Scrape, fetch, refused, fetch-failed, and internal lines redact the host for callers in context.permit_restricted. final_url stays in the response. Test captures slog records, not the journal.
401 lines
12 KiB
Go
401 lines
12 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/tls"
|
|
"encoding/json"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/netip"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.wntrmute.dev/kyle/sift/internal/auth"
|
|
"git.wntrmute.dev/kyle/sift/internal/digest"
|
|
"git.wntrmute.dev/kyle/sift/internal/fetch"
|
|
"git.wntrmute.dev/kyle/sift/internal/guard"
|
|
)
|
|
|
|
func setupAPI(t *testing.T, g *guard.Guard) (*Server, string) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
keyOut := filepath.Join(dir, "k")
|
|
keys := filepath.Join(dir, "keys.toml")
|
|
if err := auth.Add(keys, "test", "hermes-fleet", keyOut); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kb, err := os.ReadFile(keyOut)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err := auth.Load(keys)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if g == nil {
|
|
g = &guard.Guard{}
|
|
}
|
|
f := &fetch.Fetcher{Guard: g, UA: "sift-test"}
|
|
s := New("test", g, f, store, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
|
return s, strings.TrimSpace(string(kb))
|
|
}
|
|
|
|
func postJSON(t *testing.T, h http.Handler, path, key string, body any) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
b, _ := json.Marshal(body)
|
|
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
if key != "" {
|
|
req.Header.Set("Authorization", "Bearer "+key)
|
|
}
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, req)
|
|
return rec
|
|
}
|
|
|
|
func TestAuthMissingUnknown(t *testing.T) {
|
|
s, key := setupAPI(t, nil)
|
|
rec := postJSON(t, s, "/v2/scrape", "", map[string]string{"url": "https://example.com/"})
|
|
if rec.Code != 401 {
|
|
t.Fatalf("missing key: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
rec = postJSON(t, s, "/v2/scrape", "nope", map[string]string{"url": "https://example.com/"})
|
|
if rec.Code != 401 {
|
|
t.Fatalf("bad key: %d", rec.Code)
|
|
}
|
|
rec = postJSON(t, s, "/healthz", "", nil)
|
|
_ = key
|
|
}
|
|
|
|
func TestHealthzNoAuth(t *testing.T) {
|
|
s, _ := setupAPI(t, nil)
|
|
req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, req)
|
|
if rec.Code != 200 {
|
|
t.Fatal(rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestDenyPrefixThroughScrape(t *testing.T) {
|
|
g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("8.8.8.8/32")}}
|
|
s, key := setupAPI(t, g)
|
|
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://8.8.8.8/"})
|
|
if rec.Code != 403 {
|
|
t.Fatalf("got %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "blocked:") {
|
|
t.Fatal(rec.Body.String())
|
|
}
|
|
if strings.Contains(rec.Body.String(), `"success": true`) {
|
|
t.Fatal("success true")
|
|
}
|
|
}
|
|
|
|
func TestGuardLoopbackAndTailnet(t *testing.T) {
|
|
s, key := setupAPI(t, &guard.Guard{})
|
|
for _, u := range []string{
|
|
"http://127.0.0.1/",
|
|
"http://100.95.252.120:8000",
|
|
} {
|
|
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": u})
|
|
if rec.Code != 403 {
|
|
t.Fatalf("%s: %d %s", u, rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRedirectToPrivate(t *testing.T) {
|
|
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Redirect(w, r, "http://10.0.0.5/secret", http.StatusFound)
|
|
}))
|
|
defer ts.Close()
|
|
s, key := setupAPI(t, g)
|
|
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL})
|
|
if rec.Code != 403 {
|
|
t.Fatalf("got %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestSkipTLSVerificationIgnored(t *testing.T) {
|
|
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
|
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain")
|
|
_, _ = w.Write([]byte("hi"))
|
|
}))
|
|
defer ts.Close()
|
|
s, key := setupAPI(t, g)
|
|
rec := postJSON(t, s, "/v2/scrape", key, map[string]any{
|
|
"url": ts.URL,
|
|
"skipTlsVerification": true,
|
|
})
|
|
if rec.Code == 200 {
|
|
t.Fatalf("bad cert succeeded: %s", rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestDigestFallbackWhenUnset(t *testing.T) {
|
|
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html")
|
|
_, _ = w.Write([]byte("<html><body><p>hello</p></body></html>"))
|
|
}))
|
|
defer ts.Close()
|
|
s, key := setupAPI(t, g)
|
|
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "mode": "digest", "question": "what"})
|
|
if rec.Code != 200 {
|
|
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "fallback") {
|
|
t.Fatalf("want fallback: %s", rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestCapDigestFallback(t *testing.T) {
|
|
big := strings.Repeat("word ", 50000)
|
|
out, warn := capDigestFallback(big, nil)
|
|
if digest.EstTokens(out) > digest.SmallDoc {
|
|
t.Fatalf("tokens %d", digest.EstTokens(out))
|
|
}
|
|
joined := strings.Join(warn, " ")
|
|
if !strings.Contains(joined, "digest budget") {
|
|
t.Fatalf("warn %v", warn)
|
|
}
|
|
}
|
|
|
|
func TestNever502(t *testing.T) {
|
|
s, key := setupAPI(t, nil)
|
|
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})
|
|
if rec.Code == 502 {
|
|
t.Fatal("returned 502")
|
|
}
|
|
_ = key
|
|
}
|
|
|
|
func TestCleanHTMLThroughScrape(t *testing.T) {
|
|
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<!doctype html><html><head><title>Hello</title></head><body><article><h1>Hi</h1><p>World</p></article></body></html>`))
|
|
}))
|
|
defer ts.Close()
|
|
s, key := setupAPI(t, g)
|
|
rec := postJSON(t, s, "/v2/scrape", key, map[string]any{
|
|
"url": ts.URL,
|
|
"formats": []string{"markdown"},
|
|
})
|
|
if rec.Code != 200 {
|
|
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
|
}
|
|
var env map[string]any
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &env)
|
|
if env["success"] != true {
|
|
t.Fatal(env)
|
|
}
|
|
}
|
|
|
|
func TestJSONJQ(t *testing.T) {
|
|
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"a":1,"b":2}`))
|
|
}))
|
|
defer ts.Close()
|
|
s, key := setupAPI(t, g)
|
|
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": ".a", "mode": "clean"})
|
|
if rec.Code != 200 {
|
|
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "1") {
|
|
t.Fatal(rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestUnsupportedType(t *testing.T) {
|
|
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/pdf")
|
|
_, _ = w.Write([]byte("%PDF"))
|
|
}))
|
|
defer ts.Close()
|
|
s, key := setupAPI(t, g)
|
|
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL})
|
|
if rec.Code != 415 {
|
|
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestFetcherUsesSameGuard(t *testing.T) {
|
|
g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/32")}}
|
|
s, key := setupAPI(t, g)
|
|
if s.Fetch.Guard != g {
|
|
t.Fatal("fetcher not using the same Guard")
|
|
}
|
|
_ = key
|
|
_ = tls.VersionTLS13
|
|
}
|
|
|
|
func TestJQTooLarge(t *testing.T) {
|
|
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`0`))
|
|
}))
|
|
defer ts.Close()
|
|
s, key := setupAPI(t, g)
|
|
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": "range(20000)", "mode": "clean"})
|
|
if rec.Code != 400 || !strings.Contains(rec.Body.String(), "jq output too large") {
|
|
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestHTTPServerWriteTimeout(t *testing.T) {
|
|
s := New("test", &guard.Guard{}, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
|
hs := s.HTTPServer("127.0.0.1:0")
|
|
if hs.WriteTimeout != 120*time.Second {
|
|
t.Fatalf("WriteTimeout %s", hs.WriteTimeout)
|
|
}
|
|
}
|
|
|
|
type recHandler struct {
|
|
mu sync.Mutex
|
|
recs []slog.Record
|
|
}
|
|
|
|
func (h *recHandler) Enabled(context.Context, slog.Level) bool { return true }
|
|
func (h *recHandler) Handle(_ context.Context, r slog.Record) error {
|
|
h.mu.Lock()
|
|
h.recs = append(h.recs, r.Clone())
|
|
h.mu.Unlock()
|
|
return nil
|
|
}
|
|
func (h *recHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
|
|
func (h *recHandler) WithGroup(string) slog.Handler { return h }
|
|
|
|
func recHost(r slog.Record) string {
|
|
var host string
|
|
r.Attrs(func(a slog.Attr) bool {
|
|
if a.Key == "host" {
|
|
host = fmtString(a.Value)
|
|
}
|
|
return true
|
|
})
|
|
return host
|
|
}
|
|
|
|
func recMsg(r slog.Record) string { return r.Message }
|
|
|
|
func fmtString(v slog.Value) string {
|
|
if v.Kind() == slog.KindString {
|
|
return v.String()
|
|
}
|
|
return v.Any().(string)
|
|
}
|
|
|
|
func TestRestrictedCallerLogsBlankHost(t *testing.T) {
|
|
dir := t.TempDir()
|
|
keys := filepath.Join(dir, "keys.toml")
|
|
fleetOut := filepath.Join(dir, "fleet")
|
|
gsOut := filepath.Join(dir, "gs")
|
|
if err := auth.Add(keys, "test", "hermes-fleet", fleetOut); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := auth.Add(keys, "straylight-gs", "guardian-star", gsOut); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fleetKey, err := os.ReadFile(fleetOut)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gsKey, err := os.ReadFile(gsOut)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err := auth.Load(keys)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
|
|
h := &recHandler{}
|
|
s := New("test", g, &fetch.Fetcher{Guard: g, UA: "sift-test"}, store, slog.New(h))
|
|
s.PermitRestricted = []string{"straylight-gs"}
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<!doctype html><html><head><title>Hi</title></head><body><article><p>ok</p></article></body></html>`))
|
|
}))
|
|
defer ts.Close()
|
|
|
|
rec := postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": ts.URL, "mode": "clean"})
|
|
if rec.Code != 200 {
|
|
t.Fatalf("gs fetch: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
var env map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
src, _ := env["source"].(map[string]any)
|
|
final, _ := src["final_url"].(string)
|
|
if !strings.Contains(final, hostOf(ts.URL)) {
|
|
t.Fatalf("final_url still in body, got %q", final)
|
|
}
|
|
|
|
rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(gsKey)), map[string]any{"url": "http://10.0.0.5/", "mode": "clean"})
|
|
if rec.Code != 403 {
|
|
t.Fatalf("gs refused: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
rec = postJSON(t, s, "/v1/fetch", strings.TrimSpace(string(fleetKey)), map[string]any{"url": ts.URL, "mode": "clean"})
|
|
if rec.Code != 200 {
|
|
t.Fatalf("fleet fetch: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
var gsFetch, gsRefused, fleetFetch string
|
|
h.mu.Lock()
|
|
for _, r := range h.recs {
|
|
switch recMsg(r) {
|
|
case "fetch":
|
|
if attrCaller(r) == "straylight-gs" {
|
|
gsFetch = recHost(r)
|
|
}
|
|
if attrCaller(r) == "test" {
|
|
fleetFetch = recHost(r)
|
|
}
|
|
case "refused":
|
|
if attrCaller(r) == "straylight-gs" {
|
|
gsRefused = recHost(r)
|
|
}
|
|
}
|
|
}
|
|
h.mu.Unlock()
|
|
if gsFetch != "-" {
|
|
t.Fatalf("gs fetch host=%q want -", gsFetch)
|
|
}
|
|
if gsRefused != "-" {
|
|
t.Fatalf("gs refused host=%q want -", gsRefused)
|
|
}
|
|
wantHost := hostOf(ts.URL)
|
|
if fleetFetch != wantHost {
|
|
t.Fatalf("fleet fetch host=%q want %q", fleetFetch, wantHost)
|
|
}
|
|
}
|
|
|
|
func attrCaller(r slog.Record) string {
|
|
var name string
|
|
r.Attrs(func(a slog.Attr) bool {
|
|
if a.Key == "caller" {
|
|
name = a.Value.String()
|
|
}
|
|
return true
|
|
})
|
|
return name
|
|
}
|