p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification
is ignored. No 502. keys add writes the secret only to --out. Release binary
has no -test-allow-prefix.

internal/api/api.go

@@ -0,0 +1,398 @@
+package api
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/auth"
+	"git.wntrmute.dev/kyle/sift/internal/clean"
+	"git.wntrmute.dev/kyle/sift/internal/fetch"
+	"git.wntrmute.dev/kyle/sift/internal/guard"
+)
+
+const maxReqBody = 64 << 10
+
+type Server struct {
+	Version string
+	Log     *slog.Logger
+	Guard   *guard.Guard
+	Fetch   *fetch.Fetcher
+	Auth    *auth.Store
+	Mux     *http.ServeMux
+	OutCap  int
+
+	global chan struct{}
+	perMu  sync.Mutex
+	per    map[string]chan struct{}
+
+	reqN, blockN, scrapeN atomic.Uint64
+}
+
+func New(version string, g *guard.Guard, f *fetch.Fetcher, keys *auth.Store, log *slog.Logger) *Server {
+	if log == nil {
+		log = slog.Default()
+	}
+	s := &Server{
+		Version: version,
+		Log:     log,
+		Guard:   g,
+		Fetch:   f,
+		Auth:    keys,
+		Mux:     http.NewServeMux(),
+		OutCap:  200_000,
+		global:  make(chan struct{}, 8),
+		per:     map[string]chan struct{}{},
+	}
+	s.Mux.HandleFunc("GET /healthz", s.healthz)
+	s.Mux.HandleFunc("GET /healthz/", s.healthz)
+	s.Mux.HandleFunc("GET /metrics", s.metrics)
+	s.Mux.HandleFunc("POST /v2/scrape", s.scrape)
+	s.Mux.HandleFunc("POST /v1/fetch", s.v1fetch)
+	return s
+}
+
+func (s *Server) HTTPServer(addr string) *http.Server {
+	return &http.Server{
+		Addr:              addr,
+		Handler:           s,
+		ReadHeaderTimeout: 5 * time.Second,
+		ReadTimeout:       15 * time.Second,
+		WriteTimeout:      30 * time.Second,
+		IdleTimeout:       30 * time.Second,
+		MaxHeaderBytes:    16 << 10,
+	}
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxReqBody)
+	s.Mux.ServeHTTP(w, r)
+}
+
+func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]any{"status": "ok", "version": s.Version})
+}
+
+func (s *Server) metrics(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]any{
+		"requests": s.reqN.Load(),
+		"blocks":   s.blockN.Load(),
+		"scrapes":  s.scrapeN.Load(),
+	})
+}
+
+type scrapeReq struct {
+	URL                  string   `json:"url"`
+	Formats              []string `json:"formats"`
+	OnlyMainContent      *bool    `json:"onlyMainContent"`
+	SkipTlsVerification  *bool    `json:"skipTlsVerification"`
+	RemoveBase64Images   *bool    `json:"removeBase64Images"`
+	FastMode             *bool    `json:"fastMode"`
+	BlockAds             *bool    `json:"blockAds"`
+	StoreInCache         *bool    `json:"storeInCache"`
+	MaxAge               *int     `json:"maxAge"`
+	Mobile               *bool    `json:"mobile"`
+	Origin               string   `json:"origin"`
+}
+
+type v1Req struct {
+	URL      string `json:"url"`
+	Question string `json:"question"`
+	Mode     string `json:"mode"`
+	Context  *bool  `json:"context"`
+	JQ       string `json:"jq"`
+}
+
+func (s *Server) caller(w http.ResponseWriter, r *http.Request) *auth.Caller {
+	h := r.Header.Get("Authorization")
+	tok := strings.TrimSpace(strings.TrimPrefix(h, "Bearer"))
+	tok = strings.TrimSpace(tok)
+	if tok == "" || s.Auth == nil {
+		writeErr(w, 401, "unauthorized", "")
+		return nil
+	}
+	c := s.Auth.LookupBearer(tok)
+	if c == nil {
+		writeErr(w, 401, "unauthorized", "")
+		return nil
+	}
+	return c
+}
+
+func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
+	s.reqN.Add(1)
+	c := s.caller(w, r)
+	if c == nil {
+		return
+	}
+	var req scrapeReq
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.URL == "" {
+		writeErr(w, 400, "invalid url", "")
+		return
+	}
+	_ = req.SkipTlsVerification // MUST ignore; never disable TLS verify
+	onlyMain := true
+	if req.OnlyMainContent != nil {
+		onlyMain = *req.OnlyMainContent
+	}
+	wantMD, wantHTML := true, false
+	if len(req.Formats) > 0 {
+		wantMD, wantHTML = false, false
+		for _, f := range req.Formats {
+			switch strings.ToLower(f) {
+			case "markdown":
+				wantMD = true
+			case "html":
+				wantHTML = true
+			}
+		}
+		if !wantMD && !wantHTML {
+			wantMD = true
+		}
+	}
+	s.scrapeN.Add(1)
+	res, cl, ferr := s.doFetch(r, c, req.URL, "", onlyMain)
+	if ferr != nil {
+		s.writeFetchErr(w, ferr)
+		return
+	}
+	md, htmlOut := cl.Markdown, ""
+	if wantHTML {
+		htmlOut = cl.HTML
+	}
+	if !wantMD {
+		md = ""
+	}
+	rawTok := len(res.Body) / 4
+	retTok := len(md) / 4
+	s.Log.Info("scrape", "caller", c.Name, "host", hostOf(res.FinalURL), "status", res.Status, "bytes", len(res.Body))
+	writeJSON(w, 200, map[string]any{
+		"success": true,
+		"data": map[string]any{
+			"markdown": md,
+			"html":     htmlOut,
+			"metadata": map[string]any{
+				"title":       cl.Title,
+				"sourceURL":   res.FinalURL,
+				"statusCode":  res.Status,
+				"contentType": res.ContentType,
+				"sift": map[string]any{
+					"hash":      res.SHA256(),
+					"fetchedAt": res.FetchedAt.Format(time.RFC3339),
+					"cached":    false,
+					"truncated": cl.Trunc,
+					"tokens":    map[string]int{"raw": rawTok, "returned": retTok},
+				},
+			},
+		},
+	})
+}
+
+func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
+	s.reqN.Add(1)
+	c := s.caller(w, r)
+	if c == nil {
+		return
+	}
+	var req v1Req
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.URL == "" {
+		writeErr(w, 400, "invalid url", "")
+		return
+	}
+	if req.Mode == "digest" || req.Question != "" || (req.Context != nil && *req.Context) {
+		writeErr(w, 400, "not implemented until P2", "")
+		return
+	}
+	mode := req.Mode
+	if mode == "" {
+		mode = "clean"
+	}
+	if mode != "clean" && mode != "raw" {
+		writeErr(w, 400, "invalid url", "")
+		return
+	}
+	onlyMain := true
+	res, cl, ferr := s.doFetch(r, c, req.URL, req.JQ, onlyMain)
+	if ferr != nil {
+		s.writeFetchErr(w, ferr)
+		return
+	}
+	content := cl.Markdown
+	if cl.Kind == "json" {
+		content = cl.HTML // pretty json without fence
+	}
+	if mode == "raw" {
+		content = string(res.Body)
+		if len(content) > s.OutCap {
+			content = content[:s.OutCap]
+			cl.Trunc = true
+			cl.Warning = "truncated to output cap"
+		}
+	}
+	warn := []string{}
+	if cl.Warning != "" {
+		warn = append(warn, cl.Warning)
+	}
+	s.Log.Info("fetch", "caller", c.Name, "host", hostOf(res.FinalURL), "status", res.Status, "bytes", len(res.Body))
+	writeJSON(w, 200, map[string]any{
+		"content":   content,
+		"mode":      mode,
+		"untrusted": true,
+		"source": map[string]any{
+			"url":          res.URL,
+			"final_url":    res.FinalURL,
+			"status":       res.Status,
+			"content_type": res.ContentType,
+			"fetched_at":   res.FetchedAt.Format(time.RFC3339),
+			"hash":         res.SHA256(),
+			"bytes":        len(res.Body),
+			"cached":       false,
+		},
+		"warnings": warn,
+		"sizes": map[string]int{
+			"raw_tokens":      len(res.Body) / 4,
+			"returned_tokens": len(content) / 4,
+		},
+	})
+}
+
+func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, error) {
+	if err := s.acquire(c.Name, rawURL); err != nil {
+		return nil, nil, err
+	}
+	defer s.release(c.Name, rawURL)
+
+	res, err := s.Fetch.Fetch(r.Context(), rawURL)
+	if err != nil {
+		return nil, nil, err
+	}
+	kind := clean.MediaKind(res.ContentType)
+	if kind == "sniff" {
+		kind = clean.Sniff(res.Body)
+	}
+	if kind == "other" {
+		ct := res.ContentType
+		if ct == "" {
+			ct = "unknown"
+		}
+		return nil, nil, &fetch.Error{Status: 415, Hint: "browser", Msg: "unsupported content type " + ct + "; hint: browser"}
+	}
+	var cl *clean.Result
+	var cerr error
+	switch kind {
+	case "html":
+		cl, cerr = clean.HTML(res, onlyMain, s.OutCap)
+	case "json":
+		cl, cerr = clean.JSON(res, jq, s.OutCap)
+	default:
+		cl, cerr = clean.Text(res, s.OutCap)
+	}
+	if cerr != nil {
+		return nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
+	}
+	return res, cl, nil
+}
+
+func (s *Server) writeFetchErr(w http.ResponseWriter, err error) {
+	var fe *fetch.Error
+	if errors.As(err, &fe) {
+		if fe.Status == 403 {
+			s.blockN.Add(1)
+		}
+		if fe.Status == 502 {
+			fe.Status = 500
+			fe.Msg = "internal error"
+		}
+		writeErr(w, fe.Status, fe.Msg, fe.Hint)
+		return
+	}
+	s.Log.Error("internal", "err", err)
+	writeErr(w, 500, "internal error", "")
+}
+
+func (s *Server) acquire(caller, rawURL string) error {
+	host := hostOf(rawURL)
+	key := caller + "|" + host
+	select {
+	case s.global <- struct{}{}:
+	case <-time.After(5 * time.Second):
+		return &fetch.Error{Status: 429, Msg: "busy"}
+	}
+	s.perMu.Lock()
+	ch, ok := s.per[key]
+	if !ok {
+		ch = make(chan struct{}, 2)
+		s.per[key] = ch
+	}
+	s.perMu.Unlock()
+	select {
+	case ch <- struct{}{}:
+		return nil
+	case <-time.After(5 * time.Second):
+		<-s.global
+		return &fetch.Error{Status: 429, Msg: "busy"}
+	}
+}
+
+func (s *Server) release(caller, rawURL string) {
+	key := caller + "|" + hostOf(rawURL)
+	s.perMu.Lock()
+	ch := s.per[key]
+	s.perMu.Unlock()
+	if ch != nil {
+		select {
+		case <-ch:
+		default:
+		}
+	}
+	select {
+	case <-s.global:
+	default:
+	}
+}
+
+func hostOf(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return ""
+	}
+	return u.Hostname()
+}
+
+func writeErr(w http.ResponseWriter, status int, msg, hint string) {
+	if status == 502 {
+		status = 500
+		msg = "internal error"
+	}
+	body := map[string]any{"success": false, "error": msg}
+	if hint != "" {
+		body["hint"] = hint
+	}
+	writeJSON(w, status, body)
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(v)
+}
+
+func HashBody(b []byte) string {
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:])
+}
+
+func Drain(r io.Reader) {
+	_, _ = io.Copy(io.Discard, r)
+}

internal/api/api_test.go

@@ -0,0 +1,219 @@
+package api
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.wntrmute.dev/kyle/sift/internal/auth"
+	"git.wntrmute.dev/kyle/sift/internal/fetch"
+	"git.wntrmute.dev/kyle/sift/internal/guard"
+)
+
+func setupAPI(t *testing.T, g *guard.Guard) (*Server, string) {
+	t.Helper()
+	dir := t.TempDir()
+	keyOut := filepath.Join(dir, "k")
+	keys := filepath.Join(dir, "keys.toml")
+	if err := auth.Add(keys, "test", "hermes-fleet", keyOut); err != nil {
+		t.Fatal(err)
+	}
+	kb, err := os.ReadFile(keyOut)
+	if err != nil {
+		t.Fatal(err)
+	}
+	store, err := auth.Load(keys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if g == nil {
+		g = &guard.Guard{}
+	}
+	f := &fetch.Fetcher{Guard: g, UA: "sift-test"}
+	s := New("test", g, f, store, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	return s, strings.TrimSpace(string(kb))
+}
+
+func postJSON(t *testing.T, h http.Handler, path, key string, body any) *httptest.ResponseRecorder {
+	t.Helper()
+	b, _ := json.Marshal(body)
+	req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b))
+	req.Header.Set("Content-Type", "application/json")
+	if key != "" {
+		req.Header.Set("Authorization", "Bearer "+key)
+	}
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+	return rec
+}
+
+func TestAuthMissingUnknown(t *testing.T) {
+	s, key := setupAPI(t, nil)
+	rec := postJSON(t, s, "/v2/scrape", "", map[string]string{"url": "https://example.com/"})
+	if rec.Code != 401 {
+		t.Fatalf("missing key: %d %s", rec.Code, rec.Body.String())
+	}
+	rec = postJSON(t, s, "/v2/scrape", "nope", map[string]string{"url": "https://example.com/"})
+	if rec.Code != 401 {
+		t.Fatalf("bad key: %d", rec.Code)
+	}
+	rec = postJSON(t, s, "/healthz", "", nil)
+	_ = key
+}
+
+func TestHealthzNoAuth(t *testing.T) {
+	s, _ := setupAPI(t, nil)
+	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
+	rec := httptest.NewRecorder()
+	s.ServeHTTP(rec, req)
+	if rec.Code != 200 {
+		t.Fatal(rec.Code)
+	}
+}
+
+func TestDenyPrefixThroughScrape(t *testing.T) {
+	g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("8.8.8.8/32")}}
+	s, key := setupAPI(t, g)
+	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://8.8.8.8/"})
+	if rec.Code != 403 {
+		t.Fatalf("got %d %s", rec.Code, rec.Body.String())
+	}
+	if !strings.Contains(rec.Body.String(), "blocked:") {
+		t.Fatal(rec.Body.String())
+	}
+	if strings.Contains(rec.Body.String(), `"success": true`) {
+		t.Fatal("success true")
+	}
+}
+
+func TestGuardLoopbackAndTailnet(t *testing.T) {
+	s, key := setupAPI(t, &guard.Guard{})
+	for _, u := range []string{
+		"http://127.0.0.1/",
+		"http://100.95.252.120:8000",
+	} {
+		rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": u})
+		if rec.Code != 403 {
+			t.Fatalf("%s: %d %s", u, rec.Code, rec.Body.String())
+		}
+	}
+}
+
+func TestRedirectToPrivate(t *testing.T) {
+	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "http://10.0.0.5/secret", http.StatusFound)
+	}))
+	defer ts.Close()
+	s, key := setupAPI(t, g)
+	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL})
+	if rec.Code != 403 {
+		t.Fatalf("got %d %s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestSkipTLSVerificationIgnored(t *testing.T) {
+	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, _ = w.Write([]byte("hi"))
+	}))
+	defer ts.Close()
+	s, key := setupAPI(t, g)
+	rec := postJSON(t, s, "/v2/scrape", key, map[string]any{
+		"url":                 ts.URL,
+		"skipTlsVerification": true,
+	})
+	if rec.Code == 200 {
+		t.Fatalf("bad cert succeeded: %s", rec.Body.String())
+	}
+}
+
+func TestP2Rejected(t *testing.T) {
+	s, key := setupAPI(t, nil)
+	rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": "http://example.com/", "mode": "digest"})
+	if rec.Code != 400 || !strings.Contains(rec.Body.String(), "P2") {
+		t.Fatalf("%d %s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestNever502(t *testing.T) {
+	s, key := setupAPI(t, nil)
+	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})
+	if rec.Code == 502 {
+		t.Fatal("returned 502")
+	}
+	_ = key
+}
+
+func TestCleanHTMLThroughScrape(t *testing.T) {
+	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(`<!doctype html><html><head><title>Hello</title></head><body><article><h1>Hi</h1><p>World</p></article></body></html>`))
+	}))
+	defer ts.Close()
+	s, key := setupAPI(t, g)
+	rec := postJSON(t, s, "/v2/scrape", key, map[string]any{
+		"url":     ts.URL,
+		"formats": []string{"markdown"},
+	})
+	if rec.Code != 200 {
+		t.Fatalf("%d %s", rec.Code, rec.Body.String())
+	}
+	var env map[string]any
+	_ = json.Unmarshal(rec.Body.Bytes(), &env)
+	if env["success"] != true {
+		t.Fatal(env)
+	}
+}
+
+func TestJSONJQ(t *testing.T) {
+	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"a":1,"b":2}`))
+	}))
+	defer ts.Close()
+	s, key := setupAPI(t, g)
+	rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": ".a", "mode": "clean"})
+	if rec.Code != 200 {
+		t.Fatalf("%d %s", rec.Code, rec.Body.String())
+	}
+	if !strings.Contains(rec.Body.String(), "1") {
+		t.Fatal(rec.Body.String())
+	}
+}
+
+func TestUnsupportedType(t *testing.T) {
+	g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/pdf")
+		_, _ = w.Write([]byte("%PDF"))
+	}))
+	defer ts.Close()
+	s, key := setupAPI(t, g)
+	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": ts.URL})
+	if rec.Code != 415 {
+		t.Fatalf("%d %s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestFetcherUsesSameGuard(t *testing.T) {
+	g := &guard.Guard{DenyPrefixes: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/32")}}
+	s, key := setupAPI(t, g)
+	if s.Fetch.Guard != g {
+		t.Fatal("fetcher not using the same Guard")
+	}
+	_ = key
+	_ = tls.VersionTLS13
+}

internal/auth/auth.go

@@ -0,0 +1,155 @@
+package auth
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+type Caller struct {
+	Name      string `toml:"name"`
+	Workspace string `toml:"workspace"`
+	KeySHA256 string `toml:"key_sha256"`
+}
+
+type file struct {
+	Caller []Caller `toml:"caller"`
+}
+
+// Store holds hashed keys. Reload on SIGHUP via Load.
+type Store struct {
+	mu      sync.RWMutex
+	path    string
+	callers []Caller
+}
+
+func Load(path string) (*Store, error) {
+	s := &Store{path: path}
+	if err := s.reload(); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+func (s *Store) Reload() error { return s.reload() }
+
+func (s *Store) reload() error {
+	b, err := os.ReadFile(s.path)
+	if err != nil {
+		return err
+	}
+	var f file
+	if err := toml.Unmarshal(b, &f); err != nil {
+		return err
+	}
+	s.mu.Lock()
+	s.callers = f.Caller
+	s.mu.Unlock()
+	return nil
+}
+
+// LookupBearer returns the caller for a raw bearer token, or nil.
+func (s *Store) LookupBearer(token string) *Caller {
+	if token == "" || s == nil {
+		return nil
+	}
+	sum := sha256.Sum256([]byte(token))
+	want := hex.EncodeToString(sum[:])
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	var found *Caller
+	for i := range s.callers {
+		got := strings.ToLower(strings.TrimSpace(s.callers[i].KeySHA256))
+		if len(got) != len(want) {
+			continue
+		}
+		if subtle.ConstantTimeCompare([]byte(got), []byte(want)) == 1 {
+			c := s.callers[i]
+			found = &c
+		}
+	}
+	return found
+}
+
+func HashKey(key string) string {
+	sum := sha256.Sum256([]byte(key))
+	return hex.EncodeToString(sum[:])
+}
+
+func GenerateKey() (string, error) {
+	var b [32]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(b[:]), nil
+}
+
+func Add(path, name, workspace, keyOut string) error {
+	if _, err := os.Stat(keyOut); err == nil {
+		return fmt.Errorf("refusing to overwrite %s", keyOut)
+	}
+	key, err := GenerateKey()
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(keyOut, []byte(key+"\n"), 0o600); err != nil {
+		return err
+	}
+	var f file
+	if b, err := os.ReadFile(path); err == nil {
+		_ = toml.Unmarshal(b, &f)
+	}
+	f.Caller = append(f.Caller, Caller{Name: name, Workspace: workspace, KeySHA256: HashKey(key)})
+	out, err := toml.Marshal(f)
+	if err != nil {
+		return err
+	}
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, out, 0o640); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+func List(path string) ([]Caller, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var f file
+	if err := toml.Unmarshal(b, &f); err != nil {
+		return nil, err
+	}
+	return f.Caller, nil
+}
+
+func Revoke(path, name string) error {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	var f file
+	if err := toml.Unmarshal(b, &f); err != nil {
+		return err
+	}
+	out := f.Caller[:0]
+	for _, c := range f.Caller {
+		if c.Name != name {
+			out = append(out, c)
+		}
+	}
+	f.Caller = out
+	raw, err := toml.Marshal(f)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, raw, 0o640)
+}

internal/auth/auth_test.go

@@ -0,0 +1,52 @@
+package auth
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestAddLookupRevoke(t *testing.T) {
+	dir := t.TempDir()
+	keys := filepath.Join(dir, "keys.toml")
+	out := filepath.Join(dir, "k")
+	if err := Add(keys, "talos", "hermes-fleet", out); err != nil {
+		t.Fatal(err)
+	}
+	st, err := os.Stat(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.Mode().Perm()&0o077 != 0 {
+		t.Fatalf("perms %v", st.Mode())
+	}
+	b, _ := os.ReadFile(out)
+	key := string(b)
+	if key[len(key)-1] == '\n' {
+		key = key[:len(key)-1]
+	}
+	s, err := Load(keys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.LookupBearer(key) == nil || s.LookupBearer(key).Name != "talos" {
+		t.Fatal("lookup failed")
+	}
+	if s.LookupBearer("nope") != nil {
+		t.Fatal("unknown accepted")
+	}
+	if err := Add(keys, "x", "y", out); err == nil {
+		t.Fatal("overwrite allowed")
+	}
+	cs, err := List(keys)
+	if err != nil || len(cs) != 1 || cs[0].Name != "talos" {
+		t.Fatalf("%v %v", cs, err)
+	}
+	if err := Revoke(keys, "talos"); err != nil {
+		t.Fatal(err)
+	}
+	s, _ = Load(keys)
+	if s.LookupBearer(key) != nil {
+		t.Fatal("revoked still valid")
+	}
+}

internal/clean/clean.go

@@ -0,0 +1,281 @@
+package clean
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/url"
+	"strings"
+	"time"
+
+	readability "github.com/go-shiori/go-readability"
+	htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
+	"github.com/itchyny/gojq"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/charset"
+
+	"git.wntrmute.dev/kyle/sift/internal/fetch"
+)
+
+const defaultOutCap = 200_000
+
+type Result struct {
+	Markdown string
+	HTML     string
+	Title    string
+	Byline   string
+	Date     string
+	Kind     string // html, json, text
+	Warning  string
+	Trunc    bool
+}
+
+func DecodeUTF8(body []byte, contentType string) ([]byte, error) {
+	r, err := charset.NewReader(bytes.NewReader(body), contentType)
+	if err != nil {
+		return body, nil
+	}
+	out, err := io.ReadAll(r)
+	if err != nil {
+		return body, nil
+	}
+	return out, nil
+}
+
+func MediaKind(ct string) string {
+	ct = strings.ToLower(strings.TrimSpace(ct))
+	if i := strings.IndexByte(ct, ';'); i >= 0 {
+		ct = strings.TrimSpace(ct[:i])
+	}
+	switch {
+	case ct == "text/html" || ct == "application/xhtml+xml":
+		return "html"
+	case ct == "application/json" || strings.HasSuffix(ct, "+json"):
+		return "json"
+	case ct == "text/plain" || ct == "text/markdown" || ct == "text/csv":
+		return "text"
+	case ct == "":
+		return "sniff"
+	default:
+		return "other"
+	}
+}
+
+func Sniff(body []byte) string {
+	s := strings.TrimSpace(string(body[:min(len(body), 256)]))
+	ls := strings.ToLower(s)
+	if strings.HasPrefix(ls, "<!doctype html") || strings.HasPrefix(ls, "<html") {
+		return "html"
+	}
+	if strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[") {
+		return "json"
+	}
+	return "text"
+}
+
+func HTML(res *fetch.Resource, onlyMain bool, outCap int) (*Result, error) {
+	if outCap <= 0 {
+		outCap = defaultOutCap
+	}
+	body, _ := DecodeUTF8(res.Body, res.ContentType)
+	base := res.FinalURL
+	if base == "" {
+		base = res.URL
+	}
+	baseURL, _ := url.Parse(base)
+
+	title, byline, date := "", "", ""
+	var articleHTML string
+	if onlyMain {
+		pageURL := baseURL
+		if pageURL == nil {
+			pageURL, _ = url.Parse("http://localhost/")
+		}
+		art, err := readability.FromReader(bytes.NewReader(body), pageURL)
+		if err == nil && strings.TrimSpace(art.Content) != "" {
+			articleHTML = art.Content
+			title, byline = art.Title, art.Byline
+			if art.PublishedTime != nil && !art.PublishedTime.IsZero() {
+				date = art.PublishedTime.Format(time.RFC3339)
+			}
+		}
+	}
+	if articleHTML == "" {
+		articleHTML = string(body)
+		if t := extractTitle(body); t != "" {
+			title = t
+		}
+	}
+	articleHTML = stripDanger(articleHTML)
+	articleHTML = absolutize(articleHTML, baseURL)
+	md, err := htmltomarkdown.ConvertString(articleHTML)
+	if err != nil {
+		return nil, err
+	}
+	out := &Result{Markdown: md, HTML: articleHTML, Title: title, Byline: byline, Date: date, Kind: "html"}
+	return capResult(out, outCap), nil
+}
+
+func JSON(res *fetch.Resource, jqFilter string, outCap int) (*Result, error) {
+	if outCap <= 0 {
+		outCap = defaultOutCap
+	}
+	body, _ := DecodeUTF8(res.Body, res.ContentType)
+	var v any
+	if err := json.Unmarshal(body, &v); err != nil {
+		return nil, fmt.Errorf("invalid json")
+	}
+	if jqFilter != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		q, err := gojq.Parse(jqFilter)
+		if err != nil {
+			return nil, fmt.Errorf("invalid jq: %w", err)
+		}
+		code, err := gojq.Compile(q)
+		if err != nil {
+			return nil, fmt.Errorf("invalid jq: %w", err)
+		}
+		iter := code.RunWithContext(ctx, v)
+		var acc []any
+		for {
+			x, ok := iter.Next()
+			if !ok {
+				break
+			}
+			if err, ok := x.(error); ok {
+				return nil, fmt.Errorf("jq: %w", err)
+			}
+			acc = append(acc, x)
+		}
+		switch len(acc) {
+		case 0:
+			v = nil
+		case 1:
+			v = acc[0]
+		default:
+			v = acc
+		}
+	}
+	pretty, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	out := &Result{
+		Markdown: "```json\n" + string(pretty) + "\n```\n",
+		HTML:     "",
+		Kind:     "json",
+	}
+	// raw json for /v1/fetch content: store pretty in HTML field unused; Markdown is fenced.
+	out.HTML = string(pretty)
+	return capResult(out, outCap), nil
+}
+
+func Text(res *fetch.Resource, outCap int) (*Result, error) {
+	if outCap <= 0 {
+		outCap = defaultOutCap
+	}
+	body, _ := DecodeUTF8(res.Body, res.ContentType)
+	s := string(body)
+	out := &Result{Markdown: s, Kind: "text"}
+	return capResult(out, outCap), nil
+}
+
+func capResult(r *Result, capn int) *Result {
+	if len(r.Markdown) > capn {
+		r.Markdown = r.Markdown[:capn] + "\n…[truncated]"
+		r.Trunc = true
+		r.Warning = "truncated to output cap"
+	}
+	return r
+}
+
+func stripDanger(s string) string {
+	doc, err := html.Parse(strings.NewReader(s))
+	if err != nil {
+		return s
+	}
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		c := n.FirstChild
+		for c != nil {
+			next := c.NextSibling
+			if c.Type == html.ElementNode {
+				switch c.Data {
+				case "script", "style", "noscript", "iframe":
+					n.RemoveChild(c)
+					c = next
+					continue
+				}
+				for i := 0; i < len(c.Attr); {
+					a := c.Attr[i]
+					if (a.Key == "src" || a.Key == "href") && strings.HasPrefix(strings.TrimSpace(a.Val), "data:") {
+						c.Attr = append(c.Attr[:i], c.Attr[i+1:]...)
+						continue
+					}
+					i++
+				}
+			}
+			walk(c)
+			c = next
+		}
+	}
+	walk(doc)
+	var buf bytes.Buffer
+	_ = html.Render(&buf, doc)
+	return buf.String()
+}
+
+func absolutize(s string, base *url.URL) string {
+	if base == nil {
+		return s
+	}
+	doc, err := html.Parse(strings.NewReader(s))
+	if err != nil {
+		return s
+	}
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			for i, a := range n.Attr {
+				if a.Key == "href" || a.Key == "src" {
+					if u, err := base.Parse(a.Val); err == nil {
+						n.Attr[i].Val = u.String()
+					}
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	var buf bytes.Buffer
+	_ = html.Render(&buf, doc)
+	return buf.String()
+}
+
+func extractTitle(body []byte) string {
+	doc, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return ""
+	}
+	var title string
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if title != "" {
+			return
+		}
+		if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil {
+			title = strings.TrimSpace(n.FirstChild.Data)
+			return
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	return title
+}

internal/fetch/fetch.go

@@ -0,0 +1,203 @@
+package fetch
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/guard"
+)
+
+const (
+	DefaultMaxBody = 5 << 20
+	DefaultTimeout = 20 * time.Second
+)
+
+// Resource is one successful GET.
+type Resource struct {
+	URL         string
+	FinalURL    string
+	Status      int
+	ContentType string
+	Body        []byte
+	FetchedAt   time.Time
+	Hash        string
+}
+
+func (r *Resource) SHA256() string {
+	if r.Hash != "" {
+		return r.Hash
+	}
+	sum := sha256.Sum256(r.Body)
+	r.Hash = "sha256:" + hex.EncodeToString(sum[:])
+	return r.Hash
+}
+
+// Error is a fetch failure with the HTTP status sift should return to its caller.
+type Error struct {
+	Status int
+	Msg    string
+	Hint   string
+}
+
+func (e *Error) Error() string { return e.Msg }
+
+func errf(status int, hint, format string, args ...any) *Error {
+	return &Error{Status: status, Hint: hint, Msg: fmt.Sprintf(format, args...)}
+}
+
+// Fetcher is the only outbound HTTP path. Guard must be the same instance
+// that holds DenyPrefixes.
+type Fetcher struct {
+	Guard   *guard.Guard
+	UA      string
+	MaxBody int64
+	Timeout time.Duration
+}
+
+func (f *Fetcher) maxBody() int64 {
+	if f.MaxBody <= 0 {
+		return DefaultMaxBody
+	}
+	return f.MaxBody
+}
+
+func (f *Fetcher) timeout() time.Duration {
+	if f.Timeout <= 0 {
+		return DefaultTimeout
+	}
+	return f.Timeout
+}
+
+func (f *Fetcher) ua() string {
+	if f.UA != "" {
+		return f.UA
+	}
+	return "sift/dev"
+}
+
+// Fetch GETs rawURL after Guard.CheckURL, using Guard.Client() from the same instance.
+func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
+	if f == nil || f.Guard == nil {
+		return nil, errf(500, "", "internal error")
+	}
+	ctx, cancel := context.WithTimeout(ctx, f.timeout())
+	defer cancel()
+
+	if _, _, err := f.Guard.CheckURL(ctx, rawURL); err != nil {
+		return nil, mapGuard(err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, errf(400, "", "invalid url")
+	}
+	req.Header.Set("User-Agent", f.ua())
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
+	// no cookies, no auth, no Referer
+
+	client := f.Guard.Client()
+	client.Timeout = f.timeout()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, mapDial(err)
+	}
+	defer resp.Body.Close()
+
+	limit := f.maxBody()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
+	if err != nil {
+		return nil, mapDial(err)
+	}
+	if int64(len(body)) > limit {
+		return nil, errf(413, "", "too large (>%d MB decompressed)", limit>>20)
+	}
+
+	final := rawURL
+	if resp.Request != nil && resp.Request.URL != nil {
+		final = resp.Request.URL.String()
+	}
+	ct := resp.Header.Get("Content-Type")
+	if i := strings.IndexByte(ct, ';'); i >= 0 {
+		// keep full header for charset; also store raw
+	}
+	res := &Resource{
+		URL:         rawURL,
+		FinalURL:    final,
+		Status:      resp.StatusCode,
+		ContentType: ct,
+		Body:        body,
+		FetchedAt:   time.Now().UTC(),
+	}
+
+	switch {
+	case resp.StatusCode == http.StatusOK:
+		return res, nil
+	case resp.StatusCode == 403 || resp.StatusCode == 429 || resp.StatusCode == 451:
+		return res, errf(422, "browser", "upstream %d (likely bot-blocked); hint: browser", resp.StatusCode)
+	case resp.StatusCode == 503 && looksLikeChallenge(body, resp.Header):
+		return res, errf(422, "browser", "upstream 503 (likely bot-blocked); hint: browser")
+	case resp.StatusCode >= 400:
+		return res, errf(422, "", "upstream %d", resp.StatusCode)
+	default:
+		return res, nil
+	}
+}
+
+func looksLikeChallenge(body []byte, h http.Header) bool {
+	s := strings.ToLower(string(body))
+	if strings.Contains(s, "captcha") || strings.Contains(s, "cf-challenge") || strings.Contains(s, "just a moment") {
+		return true
+	}
+	if strings.Contains(strings.ToLower(h.Get("Server")), "cloudflare") {
+		return true
+	}
+	return false
+}
+
+func mapGuard(err error) error {
+	var b *guard.BlockedError
+	if errors.As(err, &b) {
+		return &Error{Status: 403, Msg: b.Error()}
+	}
+	if strings.HasPrefix(err.Error(), "blocked:") {
+		return &Error{Status: 403, Msg: err.Error()}
+	}
+	return &Error{Status: 403, Msg: "blocked: " + err.Error()}
+}
+
+func mapDial(err error) error {
+	if err == nil {
+		return nil
+	}
+	var b *guard.BlockedError
+	if errors.As(err, &b) {
+		return &Error{Status: 403, Msg: b.Error()}
+	}
+	msg := err.Error()
+	if strings.Contains(msg, "blocked:") {
+		// net.Dialer wraps Control errors
+		if i := strings.Index(msg, "blocked:"); i >= 0 {
+			return &Error{Status: 403, Msg: msg[i:]}
+		}
+	}
+	if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline") {
+		return errf(504, "", "upstream timeout after 20s")
+	}
+	if strings.Contains(msg, "tls:") || strings.Contains(msg, "certificate") || strings.Contains(msg, "x509") {
+		return errf(422, "browser", "upstream tls handshake failed")
+	}
+	var ue *url.Error
+	if errors.As(err, &ue) {
+		return mapDial(ue.Err)
+	}
+	return errf(422, "", "upstream error")
+}

internal/guard/guard.go

@@ -42,6 +42,22 @@ type Guard struct {
 	// (home WAN /32s that hairpin through the router into LAN services).
 	// Set from config deny_prefixes / -deny-prefix; never from the URL.
 	DenyPrefixes []netip.Prefix
+	// AllowPrefixes bypasses the predicate. Production config never sets this;
+	// only tests and the sifttest-tagged -test-allow-prefix flag do.
+	AllowPrefixes []netip.Prefix
+}
+
+func (g *Guard) allows(addr netip.Addr) bool {
+	if g == nil {
+		return false
+	}
+	addr = addr.Unmap()
+	for _, p := range g.AllowPrefixes {
+		if p.Contains(addr) {
+			return true
+		}
+	}
+	return false
 }
 
 func (g *Guard) lookup(ctx context.Context, host string) ([]netip.Addr, error) {
@@ -113,6 +129,9 @@ func ParseURL(raw string) (*url.URL, netip.Addr, error) {
 		}
 	}
 	if err := checkPort(port); err != nil {
+		if addr, ok := parseIPLiteral(host); ok {
+			return u, addr, err
+		}
 		return nil, netip.Addr{}, err
 	}
 	if addr, ok := parseIPLiteral(host); ok {
@@ -244,6 +263,11 @@ func (g *Guard) CheckAddr(addr netip.Addr) error {
 	addr = addr.Unmap()
 
 	if g != nil {
+		for _, p := range g.AllowPrefixes {
+			if p.Contains(addr) {
+				return nil
+			}
+		}
 		for _, p := range g.DenyPrefixes {
 			if p.Contains(addr) {
 				return blockedAddr("denied prefix", addr)
@@ -303,7 +327,11 @@ func ipv4Compatible(addr netip.Addr) (netip.Addr, bool) {
 func (g *Guard) CheckURL(ctx context.Context, raw string) (*url.URL, []netip.Addr, error) {
 	u, literal, err := ParseURL(raw)
 	if err != nil {
-		return nil, nil, err
+		if literal.IsValid() && g.allows(literal) {
+			err = nil
+		} else {
+			return nil, nil, err
+		}
 	}
 	if literal.IsValid() {
 		if err := g.CheckAddr(literal); err != nil {
@@ -338,13 +366,16 @@ func (g *Guard) Control(network, address string, _ syscall.RawConn) error {
 	if err != nil {
 		return blocked("dial address " + address)
 	}
-	if err := checkPort(port); err != nil {
-		return err
-	}
 	addr, err := netip.ParseAddr(host)
 	if err != nil {
 		return blocked("dial address not an IP " + host)
 	}
+	if g.allows(addr) {
+		return nil
+	}
+	if err := checkPort(port); err != nil {
+		return err
+	}
 	return g.CheckAddr(addr)
 }