p1: /v2/scrape, /v1/fetch, auth, CLI, contract tests

Single Fetcher.Fetch on the Guard that holds DenyPrefixes. skipTlsVerification is ignored. No 502. keys add writes the secret only to --out. Release binary has no -test-allow-prefix.
2026-09-22 21:54:01 -07:00
parent 6cb206fb27
commit 9444a35322
311 changed files with 238557 additions and 92 deletions
@@ -0,0 +1,137 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+type clientCfg struct {
+	URL     string `toml:"url"`
+	KeyFile string `toml:"key_file"`
+}
+
+func loadClient() (base, key string, err error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", "", err
+	}
+	path := filepath.Join(home, ".config", "sift", "client.toml")
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return "", "", fmt.Errorf("read %s: %w", path, err)
+	}
+	var c clientCfg
+	if err := toml.Unmarshal(b, &c); err != nil {
+		return "", "", err
+	}
+	if c.URL == "" || c.KeyFile == "" {
+		return "", "", fmt.Errorf("client.toml: url and key_file required")
+	}
+	kf := c.KeyFile
+	if strings.HasPrefix(kf, "~/") {
+		kf = filepath.Join(home, kf[2:])
+	}
+	st, err := os.Stat(kf)
+	if err != nil {
+		return "", "", err
+	}
+	if st.Mode().Perm()&0o077 != 0 {
+		return "", "", fmt.Errorf("key file %s is group/world readable", kf)
+	}
+	kb, err := os.ReadFile(kf)
+	if err != nil {
+		return "", "", err
+	}
+	return strings.TrimRight(c.URL, "/"), strings.TrimSpace(string(kb)), nil
+}
+
+func fetchMain() {
+	fs := flag.NewFlagSet("fetch", flag.ExitOnError)
+	mode := fs.String("mode", "clean", "clean|raw")
+	jq := fs.String("jq", "", "jq filter (JSON responses)")
+	noMain := fs.Bool("no-main", false, "do not isolate main content")
+	asJSON := fs.Bool("json", false, "print full /v1/fetch envelope")
+	_ = noMain
+	_ = fs.Parse(os.Args[1:])
+	if fs.NArg() < 1 {
+		fmt.Fprintln(os.Stderr, "usage: sift fetch <url> [--mode clean|raw] [--jq FILTER] [--json]")
+		os.Exit(2)
+	}
+	rawURL := fs.Arg(0)
+	base, key, err := loadClient()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	body, _ := json.Marshal(map[string]any{
+		"url":  rawURL,
+		"mode": *mode,
+		"jq":   *jq,
+	})
+	req, err := http.NewRequest(http.MethodPost, base+"/v1/fetch", bytes.NewReader(body))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	req.Header.Set("Authorization", "Bearer "+key)
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	defer resp.Body.Close()
+	var env map[string]any
+	if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	if resp.StatusCode != 200 {
+		msg, _ := env["error"].(string)
+		hint, _ := env["hint"].(string)
+		if hint != "" {
+			fmt.Fprintf(os.Stderr, "%s\nhint: %s\n", msg, hint)
+		} else {
+			fmt.Fprintln(os.Stderr, msg)
+		}
+		os.Exit(exitFor(resp.StatusCode, msg))
+	}
+	if *asJSON {
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		_ = enc.Encode(env)
+		return
+	}
+	if c, ok := env["content"].(string); ok {
+		fmt.Print(c)
+		if !strings.HasSuffix(c, "\n") {
+			fmt.Println()
+		}
+	}
+}
+
+func exitFor(status int, msg string) int {
+	switch status {
+	case 403:
+		return 2
+	case 415:
+		return 4
+	case 401, 500:
+		return 5
+	case 422, 413, 504, 429:
+		return 3
+	default:
+		if strings.Contains(msg, "blocked:") {
+			return 2
+		}
+		return 1
+	}
+}
@@ -0,0 +1,16 @@
+package main
+
+import (
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestReleaseHelpHasNoTestAllow(t *testing.T) {
+	cmd := exec.Command("go", "run", ".", "serve", "-h")
+	cmd.Dir = "."
+	out, _ := cmd.CombinedOutput()
+	if strings.Contains(string(out), "test-allow-prefix") {
+		t.Fatalf("release help mentions test-allow-prefix:\n%s", out)
+	}
+}
@@ -0,0 +1,75 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"git.wntrmute.dev/kyle/sift/internal/auth"
+)
+
+func keysMain() {
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "usage: sift keys add|list|revoke")
+		os.Exit(2)
+	}
+	sub := os.Args[1]
+	rest := os.Args[2:]
+	switch sub {
+	case "add":
+		keysAdd(rest)
+	case "list":
+		keysList(rest)
+	case "revoke":
+		keysRevoke(rest)
+	default:
+		fmt.Fprintf(os.Stderr, "sift keys: unknown %q\n", sub)
+		os.Exit(2)
+	}
+}
+
+func keysAdd(args []string) {
+	fs := flag.NewFlagSet("keys add", flag.ExitOnError)
+	caller := fs.String("caller", "", "caller name")
+	ws := fs.String("workspace", "hermes-fleet", "Honcho workspace")
+	keys := fs.String("keys", "/var/lib/sift/keys.toml", "keys.toml path")
+	out := fs.String("out", "", "write the secret key here (0600, must not exist)")
+	_ = fs.Parse(args)
+	if *caller == "" || *out == "" {
+		fmt.Fprintln(os.Stderr, "sift keys add --caller NAME --out FILE [--keys PATH] [--workspace WS]")
+		os.Exit(2)
+	}
+	if err := auth.Add(*keys, *caller, *ws, *out); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func keysList(args []string) {
+	fs := flag.NewFlagSet("keys list", flag.ExitOnError)
+	keys := fs.String("keys", "/var/lib/sift/keys.toml", "keys.toml path")
+	_ = fs.Parse(args)
+	cs, err := auth.List(*keys)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	for _, c := range cs {
+		fmt.Printf("%s\t%s\n", c.Name, c.Workspace)
+	}
+}
+
+func keysRevoke(args []string) {
+	fs := flag.NewFlagSet("keys revoke", flag.ExitOnError)
+	caller := fs.String("caller", "", "caller name")
+	keys := fs.String("keys", "/var/lib/sift/keys.toml", "keys.toml path")
+	_ = fs.Parse(args)
+	if *caller == "" {
+		fmt.Fprintln(os.Stderr, "sift keys revoke --caller NAME")
+		os.Exit(2)
+	}
+	if err := auth.Revoke(*keys, *caller); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
@@ -1,95 +1,45 @@
 package main

 import (
-	"flag"
 	"fmt"
-	"log/slog"
-	"net"
-	"net/netip"
 	"os"
-	"os/signal"
-	"strings"
-	"syscall"
-
-	"git.wntrmute.dev/kyle/sift/internal/guard"
-	"git.wntrmute.dev/kyle/sift/internal/server"
 )

 // version is set by ldflags: -X main.version=…
 var version = "dev"

-func defaultConfigPath() string {
-	if p := os.Getenv("SIFT_CONFIG"); p != "" {
-		return p
-	}
-	return "/etc/sift/config.toml"
-}
-
-type denyFlags []netip.Prefix
-
-func (d *denyFlags) String() string {
-	var b []string
-	for _, p := range *d {
-		b = append(b, p.String())
-	}
-	return strings.Join(b, ",")
-}
-
-func (d *denyFlags) Set(s string) error {
-	p, err := netip.ParsePrefix(s)
-	if err != nil {
-		return err
-	}
-	*d = append(*d, p)
-	return nil
-}
-
 func main() {
-	listen := flag.String("listen", "127.0.0.1:8890", "listen address")
-	configPath := flag.String("config", defaultConfigPath(), "config file (optional in P0)")
-	showVersion := flag.Bool("version", false, "print version and exit")
-	var deny denyFlags
-	flag.Var(&deny, "deny-prefix", "extra CIDR to block (repeatable; home WAN /32s)")
-	flag.Parse()
-
-	if *showVersion {
+	if len(os.Args) < 2 {
+		usage()
+		os.Exit(2)
+	}
+	cmd := os.Args[1]
+	os.Args = append([]string{os.Args[0]}, os.Args[2:]...)
+	switch cmd {
+	case "serve":
+		serveMain()
+	case "fetch":
+		fetchMain()
+	case "keys":
+		keysMain()
+	case "version", "-version", "--version":
 		fmt.Println(version)
-		os.Exit(0)
-	}
-
-	log := slog.New(slog.NewTextHandler(os.Stdout, nil))
-	if _, err := os.Stat(*configPath); err == nil {
-		log.Info("config file present but unused in P0", "path", *configPath)
-	}
-	_ = &guard.Guard{DenyPrefixes: deny} // held for fetch in P1; validated at flag parse
-	if n := len(deny); n > 0 {
-		log.Info("deny prefixes loaded", "count", n)
-	}
-
-	srv := server.New(version)
-	hs := srv.HTTPServer(*listen)
-
-	ln, err := net.Listen("tcp", *listen)
-	if err != nil {
-		log.Error("listen", "err", err)
-		os.Exit(1)
-	}
-	log.Info("sift listening", "addr", ln.Addr().String(), "version", version)
-
-	errCh := make(chan error, 1)
-	go func() {
-		errCh <- hs.Serve(ln)
-	}()
-
-	sig := make(chan os.Signal, 1)
-	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
-	select {
-	case <-sig:
-		_ = hs.Close()
-	case err := <-errCh:
-		if err != nil {
-			log.Error("serve", "err", err)
-			os.Exit(1)
-		}
+	case "-h", "--help", "help":
+		usage()
+	default:
+		fmt.Fprintf(os.Stderr, "sift: unknown command %q\n", cmd)
+		usage()
+		os.Exit(2)
 	}
 }
+
+func usage() {
+	fmt.Fprintf(os.Stderr, `sift %s — LLM-oriented fetch
+
+Commands:
+  sift serve    [flags]     run the service
+  sift fetch    <url>       fetch via the service
+  sift keys     add|list|revoke
+  sift version
+`, version)
+}
@@ -0,0 +1,106 @@
+package main
+
+import (
+	"flag"
+	"log/slog"
+	"net"
+	"net/netip"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"git.wntrmute.dev/kyle/sift/internal/api"
+	"git.wntrmute.dev/kyle/sift/internal/auth"
+	"git.wntrmute.dev/kyle/sift/internal/fetch"
+	"git.wntrmute.dev/kyle/sift/internal/guard"
+)
+
+type denyFlags []netip.Prefix
+
+func (d *denyFlags) String() string {
+	var b []string
+	for _, p := range *d {
+		b = append(b, p.String())
+	}
+	return strings.Join(b, ",")
+}
+
+func (d *denyFlags) Set(s string) error {
+	p, err := netip.ParsePrefix(s)
+	if err != nil {
+		return err
+	}
+	*d = append(*d, p)
+	return nil
+}
+
+func defaultConfigPath() string {
+	if p := os.Getenv("SIFT_CONFIG"); p != "" {
+		return p
+	}
+	return "/etc/sift/config.toml"
+}
+
+func serveMain() {
+	fs := flag.NewFlagSet("serve", flag.ExitOnError)
+	listen := fs.String("listen", "127.0.0.1:8890", "listen address")
+	configPath := fs.String("config", defaultConfigPath(), "config file")
+	keysPath := fs.String("keys", "/var/lib/sift/keys.toml", "caller keys file")
+	ua := fs.String("user-agent", "sift/"+version, "outbound User-Agent")
+	var deny denyFlags
+	fs.Var(&deny, "deny-prefix", "extra CIDR to block (repeatable)")
+	g := &guard.Guard{}
+	registerTestFlags(fs, g)
+	_ = fs.Parse(os.Args[1:])
+
+	g.DenyPrefixes = deny
+	log := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	if n := len(deny); n > 0 {
+		log.Info("deny prefixes loaded", "count", n)
+	}
+	_ = configPath
+
+	store, err := auth.Load(*keysPath)
+	if err != nil {
+		log.Error("keys", "err", err, "path", *keysPath)
+		os.Exit(1)
+	}
+
+	fetcher := &fetch.Fetcher{Guard: g, UA: *ua}
+	srv := api.New(version, g, fetcher, store, log)
+	hs := srv.HTTPServer(*listen)
+
+	ln, err := net.Listen("tcp", *listen)
+	if err != nil {
+		log.Error("listen", "err", err)
+		os.Exit(1)
+	}
+	log.Info("sift listening", "addr", ln.Addr().String(), "version", version)
+
+	go func() {
+		ch := make(chan os.Signal, 1)
+		signal.Notify(ch, syscall.SIGHUP)
+		for range ch {
+			if err := store.Reload(); err != nil {
+				log.Error("keys reload", "err", err)
+			} else {
+				log.Info("keys reloaded")
+			}
+		}
+	}()
+
+	errCh := make(chan error, 1)
+	go func() { errCh <- hs.Serve(ln) }()
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
+	select {
+	case <-sig:
+		_ = hs.Close()
+	case err := <-errCh:
+		if err != nil {
+			log.Error("serve", "err", err)
+			os.Exit(1)
+		}
+	}
+}
@@ -0,0 +1,11 @@
+//go:build !sifttest
+
+package main
+
+import (
+	"flag"
+
+	"git.wntrmute.dev/kyle/sift/internal/guard"
+)
+
+func registerTestFlags(_ *flag.FlagSet, _ *guard.Guard) {}
@@ -0,0 +1,35 @@
+//go:build sifttest
+
+package main
+
+import (
+	"flag"
+	"net/netip"
+
+	"git.wntrmute.dev/kyle/sift/internal/guard"
+)
+
+type allowFlags []netip.Prefix
+
+func (d *allowFlags) String() string { return "" }
+func (d *allowFlags) Set(s string) error {
+	p, err := netip.ParsePrefix(s)
+	if err != nil {
+		return err
+	}
+	*d = append(*d, p)
+	return nil
+}
+
+func registerTestFlags(fs *flag.FlagSet, g *guard.Guard) {
+	var a allowFlags
+	fs.Func("test-allow-prefix", "TEST ONLY: extra CIDR to allow", func(s string) error {
+		p, err := netip.ParsePrefix(s)
+		if err != nil {
+			return err
+		}
+		g.AllowPrefixes = append(g.AllowPrefixes, p)
+		a = append(a, p)
+		return nil
+	})
+}