diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8ca64e5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/bin/ +/result +/result-* +*.test +/sift diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5463ea3 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +# Makefile is the source of truth for build flags. The flake's buildGoModule +# uses the same -trimpath / ldflags, and `nix flake check` runs `make test lint`. + +PREFIX ?= /usr/local +DESTDIR ?= +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +LDFLAGS := -s -w -X main.version=$(VERSION) +CGO_ENABLED ?= 0 + +GO ?= go +STATICCHECK ?= $(shell command -v staticcheck 2>/dev/null) + +.PHONY: all build test lint contract install clean + +all: build + +build: + CGO_ENABLED=$(CGO_ENABLED) $(GO) build -trimpath -ldflags "$(LDFLAGS)" -o bin/sift ./cmd/sift + +test: + $(GO) test -race ./... + +lint: + $(GO) vet ./... + @if [ -n "$(STATICCHECK)" ]; then $(STATICCHECK) ./...; else echo "staticcheck not found; skipped"; fi + +# Hermes 0.19/0.21 parser harness lands in P1. +contract: + @echo "P0: no contract tests yet" + +install: build + install -d $(DESTDIR)$(PREFIX)/bin + install -m 755 bin/sift $(DESTDIR)$(PREFIX)/bin/sift + install -d $(DESTDIR)/etc/sift + if [ ! -f $(DESTDIR)/etc/sift/config.toml ]; then \ + install -m 644 dist/sift.toml.example $(DESTDIR)/etc/sift/config.toml; \ + fi + install -d $(DESTDIR)$(PREFIX)/lib/systemd/system + install -m 644 dist/sift.service $(DESTDIR)$(PREFIX)/lib/systemd/system/sift.service + +clean: + rm -rf bin/ + rm -f sift *.test diff --git a/README.md b/README.md index a7b9786..72d76a6 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,19 @@ LLM-oriented fetch and search for the Hermes fleet: fetch a resource and get it compact, and with useful context attached — using local models (Dixie), SearXNG and Brave on orion, and Honcho for retrieval. -Status: planning. See [`docs/PLAN.md`](docs/PLAN.md). +Status: P0 skeleton (guard, `/healthz`, flake, NixOS module, Makefile). See [`docs/PLAN.md`](docs/PLAN.md). + +## Build + +Makefile is the source of truth for flags (`CGO_ENABLED=0`, `-trimpath`, `-s -w -X main.version=`). + +``` +make build # bin/sift +make test # go test -race ./... +make lint # go vet, and staticcheck if installed +make install # PREFIX=/usr/local; example config in /etc/sift/ +nix build # packages.sift, same ldflags +nix flake check # make test lint +``` + +The service does not assume Nix store paths. Non-Nix hosts: `make install` plus `dist/sift.service`. diff --git a/cmd/sift/main.go b/cmd/sift/main.go new file mode 100644 index 0000000..31045d2 --- /dev/null +++ b/cmd/sift/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "flag" + "fmt" + "log/slog" + "net" + "os" + "os/signal" + "syscall" + + "git.wntrmute.dev/kyle/sift/internal/server" +) + +// version is set by ldflags: -X main.version=… +var version = "dev" + +func defaultConfigPath() string { + if p := os.Getenv("SIFT_CONFIG"); p != "" { + return p + } + return "/etc/sift/config.toml" +} + +func main() { + listen := flag.String("listen", "127.0.0.1:8890", "listen address") + configPath := flag.String("config", defaultConfigPath(), "config file (optional in P0)") + showVersion := flag.Bool("version", false, "print version and exit") + flag.Parse() + + if *showVersion { + fmt.Println(version) + os.Exit(0) + } + + log := slog.New(slog.NewTextHandler(os.Stdout, nil)) + if _, err := os.Stat(*configPath); err == nil { + log.Info("config file present but unused in P0", "path", *configPath) + } + + srv := server.New(version) + hs := srv.HTTPServer(*listen) + + ln, err := net.Listen("tcp", *listen) + if err != nil { + log.Error("listen", "err", err) + os.Exit(1) + } + log.Info("sift listening", "addr", ln.Addr().String(), "version", version) + + errCh := make(chan error, 1) + go func() { + errCh <- hs.Serve(ln) + }() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + select { + case <-sig: + _ = hs.Close() + case err := <-errCh: + if err != nil { + log.Error("serve", "err", err) + os.Exit(1) + } + } +} diff --git a/dist/sift.service b/dist/sift.service new file mode 100644 index 0000000..5d5f190 --- /dev/null +++ b/dist/sift.service @@ -0,0 +1,41 @@ +[Unit] +Description=sift LLM-oriented fetch and search +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=sift +Group=sift +ExecStart=/usr/local/bin/sift -listen 127.0.0.1:8890 -config /etc/sift/config.toml +WorkingDirectory=/var/lib/sift +Restart=on-failure +RestartSec=5s + +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +ProtectHostname=true +ProtectClock=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +RestrictNamespaces=true +RestrictRealtime=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallFilter=~@privileged +CapabilityBoundingSet= +AmbientCapabilities= +ReadWritePaths=/var/lib/sift +ConfigurationDirectory=sift +StateDirectory=sift + +[Install] +WantedBy=multi-user.target diff --git a/dist/sift.toml.example b/dist/sift.toml.example new file mode 100644 index 0000000..d38cd07 --- /dev/null +++ b/dist/sift.toml.example @@ -0,0 +1,13 @@ +# sift configuration. P0 only uses -listen; remaining keys are reserved for P1+. +# This file is an example: the service does not assume Nix store paths. + +listen = "127.0.0.1:8890" + +# [upstreams] +# searxng = "http://127.0.0.1:8888" +# dixie = "http://dixie.scylla-hammerhead.ts.net:11434/v1" +# honcho = "http://rift.scylla-hammerhead.ts.net:8000" + +# [digest] +# model = "ornith-1.5-9b-uncensored" +# thinking = false diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..3d79727 --- /dev/null +++ b/flake.lock @@ -0,0 +1,25 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1789998861, + "narHash": "sha256-GiMDMZpYQ2eK3OFtqj/yXpRZwqfu6JfG5WWrUHWyF8M=", + "rev": "1e8bc658fc985ef27ccd66d107d767b32bb7ef98", + "type": "tarball", + "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.10402.1e8bc658fc98/nixexprs.tar.zst" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-26.05", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..c277b6d --- /dev/null +++ b/flake.nix @@ -0,0 +1,112 @@ +{ + description = "sift: LLM-oriented fetch and search"; + + inputs.nixpkgs.url = "nixpkgs/nixos-26.05"; + + outputs = + { self, nixpkgs }: + let + systems = [ "x86_64-linux" ]; + forAllSystems = nixpkgs.lib.genAttrs systems; + version = + if self ? shortRev then + self.shortRev + else if self ? dirtyShortRev then + self.dirtyShortRev + else + "dev"; + # Keep in lockstep with Makefile LDFLAGS / -trimpath. + ldflags = [ + "-s" + "-w" + "-X" + "main.version=${version}" + ]; + in + { + packages = forAllSystems ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + sift = pkgs.buildGoModule { + pname = "sift"; + inherit version; + src = ./.; + vendorHash = null; + env.CGO_ENABLED = "0"; + ldflags = ldflags; + subPackages = [ "cmd/sift" ]; + # buildGoModule already passes -trimpath. + meta = { + description = "LLM-oriented fetch and search"; + mainProgram = "sift"; + }; + }; + in + { + inherit sift; + default = sift; + } + ); + + nixosModules.sift = + { pkgs, ... }: + { + imports = [ ./nix/module.nix ]; + services.sift.package = nixpkgs.lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.sift; + }; + nixosModules.default = self.nixosModules.sift; + + checks = forAllSystems ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + src = ./.; + in + { + test = pkgs.stdenv.mkDerivation { + name = "sift-make-test-lint"; + inherit src; + nativeBuildInputs = [ + pkgs.go + pkgs.gcc + pkgs.gnumake + pkgs.git + ]; + dontConfigure = true; + buildPhase = '' + export HOME=$TMPDIR + export GOCACHE=$TMPDIR/go-cache + export GOPATH=$TMPDIR/go + export GO111MODULE=on + # race detector needs cgo; build stays CGO_ENABLED=0 in packages.sift + export CGO_ENABLED=1 + make test lint + ''; + installPhase = '' + mkdir -p $out + echo ok > $out/ok + ''; + }; + } + ); + + devShells = forAllSystems ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + default = pkgs.mkShell { + nativeBuildInputs = [ + pkgs.go + pkgs.gopls + pkgs.go-tools + pkgs.gnumake + pkgs.git + ]; + }; + } + ); + }; +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..321311c --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module git.wntrmute.dev/kyle/sift + +go 1.24 diff --git a/internal/guard/guard.go b/internal/guard/guard.go new file mode 100644 index 0000000..5068d4b --- /dev/null +++ b/internal/guard/guard.go @@ -0,0 +1,404 @@ +// Package guard is the SSRF control plane for sift's outbound fetches. +// +// Threat model: the URL is attacker-influenced (prompt-injected pages tell +// agents to fetch things). Sift runs inside the tailnet, so without these +// checks it is an SSRF gadget into Honcho, Dixie, Mattermost, orion itself, +// and the LAN. +// +// Defence is two-layered and both layers must fire: +// +// 1. Pre-check: parse the URL, require http(s) / no userinfo / port 80 or 443, +// resolve every A/AAAA, and reject the URL if *any* record is blocked. +// 2. Dial-time: net.Dialer.Control sees the literal IP about to be connect()ed +// (including Happy Eyeballs races and a rebinding answer that arrived after +// the pre-check) and applies the same address predicate. +// +// Redirects are followed manually (max 5); each hop re-runs the pre-check. +// There is no proxy, including from the environment. +package guard + +import ( + "context" + "fmt" + "net" + "net/http" + "net/netip" + "net/url" + "strconv" + "strings" + "syscall" + "time" +) + +var ( + cgnat = netip.MustParsePrefix("100.64.0.0/10") // tailnet IPv4 (CGNAT) + tailnetULA = netip.MustParsePrefix("fd7a:115c:a1e0::/48") // tailnet IPv6 + nat64 = netip.MustParsePrefix("64:ff9b::/96") + ipv4zero = netip.MustParsePrefix("0.0.0.0/8") +) + +const maxRedirects = 5 + +// LookupIP resolves a hostname to addresses. Tests inject this. +type LookupIP func(ctx context.Context, host string) ([]netip.Addr, error) + +// Guard holds the SSRF policy. A zero Guard is ready to use. +type Guard struct { + LookupIP LookupIP +} + +func (g *Guard) lookup(ctx context.Context, host string) ([]netip.Addr, error) { + if g.LookupIP != nil { + return g.LookupIP(ctx, host) + } + return net.DefaultResolver.LookupNetIP(ctx, "ip", host) +} + +// BlockedError is returned for every policy rejection. Error() is stable +// ("blocked: …") so the HTTP layer can put it in Firecrawl-shaped responses +// without agents retrying. +type BlockedError struct { + Reason string + Addr netip.Addr + Port string +} + +func (e *BlockedError) Error() string { + if e.Addr.IsValid() { + return fmt.Sprintf("blocked: %s %s", e.Reason, e.Addr) + } + if e.Port != "" { + return fmt.Sprintf("blocked: %s %s", e.Reason, e.Port) + } + return "blocked: " + e.Reason +} + +func blocked(reason string) error { + return &BlockedError{Reason: reason} +} + +func blockedAddr(reason string, addr netip.Addr) error { + return &BlockedError{Reason: reason, Addr: addr} +} + +func blockedPort(port string) error { + return &BlockedError{Reason: "port", Port: port} +} + +// ParseURL validates scheme, userinfo, host, and port. It does not resolve. +// Hostnames are returned as ASCII (non-ASCII IDN is rejected in P0: fail closed, +// punycode required). IP literals include inet_aton-style decimal/octal/hex +// forms so those never reach DNS. +func ParseURL(raw string) (*url.URL, netip.Addr, error) { + if raw == "" { + return nil, netip.Addr{}, blocked("empty url") + } + u, err := url.Parse(raw) + if err != nil { + return nil, netip.Addr{}, blocked("invalid url") + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, netip.Addr{}, blocked("scheme " + u.Scheme) + } + if u.User != nil { + return nil, netip.Addr{}, blocked("userinfo") + } + host := u.Hostname() + if host == "" { + return nil, netip.Addr{}, blocked("missing host") + } + port := u.Port() + if port == "" { + if u.Scheme == "https" { + port = "443" + } else { + port = "80" + } + } + if err := checkPort(port); err != nil { + return nil, netip.Addr{}, err + } + if addr, ok := parseIPLiteral(host); ok { + return u, addr, nil + } + if err := checkASCIIHost(host); err != nil { + return nil, netip.Addr{}, err + } + return u, netip.Addr{}, nil +} + +func checkPort(port string) error { + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 { + return blockedPort(port) + } + if n != 80 && n != 443 { + return blockedPort(port) + } + return nil +} + +func checkASCIIHost(host string) error { + for _, r := range host { + if r > 127 { + return blocked("non-ascii host (punycode required)") + } + if r < 0x20 || strings.ContainsRune(" /\\", r) { + return blocked("invalid host") + } + } + if strings.Contains(host, "..") { + return blocked("invalid host") + } + return nil +} + +// parseIPLiteral accepts canonical IPs plus inet_aton-style IPv4 (decimal, +// octal, hex, 1–4 dotted parts) and IPv4-mapped / zone-id IPv6. +func parseIPLiteral(host string) (netip.Addr, bool) { + if addr, err := netip.ParseAddr(host); err == nil { + return addr, true + } + if addr, ok := parseIPv4AtOn(host); ok { + return addr, true + } + return netip.Addr{}, false +} + +func parseIPv4AtOn(s string) (netip.Addr, bool) { + parts := strings.Split(s, ".") + if len(parts) < 1 || len(parts) > 4 { + return netip.Addr{}, false + } + nums := make([]uint64, len(parts)) + for i, p := range parts { + n, ok := parseIPv4Part(p) + if !ok { + return netip.Addr{}, false + } + nums[i] = n + } + var b [4]byte + switch len(nums) { + case 1: + if nums[0] > 0xffffffff { + return netip.Addr{}, false + } + b[0] = byte(nums[0] >> 24) + b[1] = byte(nums[0] >> 16) + b[2] = byte(nums[0] >> 8) + b[3] = byte(nums[0]) + case 2: + if nums[0] > 0xff || nums[1] > 0xffffff { + return netip.Addr{}, false + } + b[0] = byte(nums[0]) + b[1] = byte(nums[1] >> 16) + b[2] = byte(nums[1] >> 8) + b[3] = byte(nums[1]) + case 3: + if nums[0] > 0xff || nums[1] > 0xff || nums[2] > 0xffff { + return netip.Addr{}, false + } + b[0] = byte(nums[0]) + b[1] = byte(nums[1]) + b[2] = byte(nums[2] >> 8) + b[3] = byte(nums[2]) + case 4: + for i := 0; i < 4; i++ { + if nums[i] > 0xff { + return netip.Addr{}, false + } + b[i] = byte(nums[i]) + } + } + return netip.AddrFrom4(b), true +} + +func parseIPv4Part(s string) (uint64, bool) { + if s == "" { + return 0, false + } + switch { + case len(s) > 2 && (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')): + n, err := strconv.ParseUint(s[2:], 16, 32) + return n, err == nil + case len(s) > 1 && s[0] == '0': + n, err := strconv.ParseUint(s, 8, 32) + return n, err == nil + default: + n, err := strconv.ParseUint(s, 10, 32) + return n, err == nil + } +} + +// CheckAddr is the address predicate applied to every resolved record and to +// the literal IP in Dialer.Control. +func CheckAddr(addr netip.Addr) error { + if !addr.IsValid() { + return blocked("invalid address") + } + addr = addr.Unmap() + + switch { + case addr.IsUnspecified(): + return blockedAddr("unspecified address", addr) + case addr.IsLoopback(): + return blockedAddr("loopback address", addr) + case addr.IsPrivate(): + return blockedAddr("private address", addr) + case addr.IsMulticast(), addr.IsInterfaceLocalMulticast(), addr.IsLinkLocalMulticast(): + return blockedAddr("multicast address", addr) + case addr.IsLinkLocalUnicast(): + return blockedAddr("link-local address", addr) + } + + if nat64.Contains(addr) { + var b [4]byte + raw := addr.As16() + copy(b[:], raw[12:]) + embedded := netip.AddrFrom4(b) + if err := CheckAddr(embedded); err != nil { + return blockedAddr("NAT64-embedded", embedded) + } + return blockedAddr("NAT64 prefix", addr) + } + + if v4, ok := ipv4Compatible(addr); ok { + return CheckAddr(v4) + } + + if !addr.IsGlobalUnicast() { + return blockedAddr("non-global address", addr) + } + if addr.Is4() && ipv4zero.Contains(addr) { + return blockedAddr("unspecified address", addr) + } + if addr.Is4() && cgnat.Contains(addr) { + return blockedAddr("tailnet address", addr) + } + if addr.Is6() && tailnetULA.Contains(addr) { + return blockedAddr("tailnet address", addr) + } + return nil +} + +// ipv4Compatible reports deprecated IPv4-compatible IPv6 (::/96). Unspecified +// and loopback are handled before this is called. IPv4-mapped addresses are +// Unmap'd first. +func ipv4Compatible(addr netip.Addr) (netip.Addr, bool) { + if !addr.Is6() || addr.Is4In6() { + return netip.Addr{}, false + } + a := addr.As16() + for i := 0; i < 12; i++ { + if a[i] != 0 { + return netip.Addr{}, false + } + } + v4 := netip.AddrFrom4([4]byte{a[12], a[13], a[14], a[15]}) + if !v4.IsValid() || v4.IsUnspecified() { + return netip.Addr{}, false + } + return v4, true +} + +// CheckURL parses, and if the host is a name, resolves every record. Any +// blocked address fails the whole URL (mixed-record trick). +func (g *Guard) CheckURL(ctx context.Context, raw string) (*url.URL, []netip.Addr, error) { + u, literal, err := ParseURL(raw) + if err != nil { + return nil, nil, err + } + if literal.IsValid() { + if err := CheckAddr(literal); err != nil { + return nil, nil, err + } + return u, []netip.Addr{literal}, nil + } + addrs, err := g.lookup(ctx, u.Hostname()) + if err != nil { + return nil, nil, fmt.Errorf("blocked: resolve %s: %w", u.Hostname(), err) + } + if len(addrs) == 0 { + return nil, nil, blocked("no addresses for " + u.Hostname()) + } + for _, a := range addrs { + if err := CheckAddr(a); err != nil { + return nil, nil, err + } + } + return u, addrs, nil +} + +// Control is a net.Dialer Control / ControlContext hook. address is the +// remote IP:port about to be connect()ed. +func (g *Guard) Control(network, address string, _ syscall.RawConn) error { + host, port, err := net.SplitHostPort(address) + if err != nil { + return blocked("dial address " + address) + } + if err := checkPort(port); err != nil { + return err + } + addr, err := netip.ParseAddr(host) + if err != nil { + return blocked("dial address not an IP " + host) + } + return CheckAddr(addr) +} + +// ControlContext is the context-aware form; policy does not use ctx today. +func (g *Guard) ControlContext(_ context.Context, network, address string, c syscall.RawConn) error { + return g.Control(network, address, c) +} + +// CheckRedirect is an http.Client CheckRedirect hook. Max 5 hops; every hop +// is re-validated (parse + resolve + address predicate). +func (g *Guard) CheckRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= maxRedirects { + return blocked("too many redirects") + } + if req.URL == nil { + return blocked("redirect missing url") + } + _, _, err := g.CheckURL(req.Context(), req.URL.String()) + return err +} + +// Dialer returns a net.Dialer whose Control hook is the address predicate. +// Connect timeout is 5s as specified in the plan. +func (g *Guard) Dialer() *net.Dialer { + return &net.Dialer{ + Timeout: 5 * time.Second, + Control: g.Control, + ControlContext: g.ControlContext, + } +} + +// Transport is an HTTP transport with no proxy, the guarded dialer, and a +// 20s overall request timeout left to the caller via context. Body size caps +// land in P1. +func (g *Guard) Transport() *http.Transport { + d := g.Dialer() + return &http.Transport{ + Proxy: nil, // never ProxyFromEnvironment + DialContext: d.DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 8, + IdleConnTimeout: 30 * time.Second, + TLSHandshakeTimeout: 5 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DisableCompression: false, + } +} + +// Client returns an http.Client that refuses blocked URLs on redirect. +// Callers must still CheckURL the initial URL; Client.CheckRedirect covers hops. +func (g *Guard) Client() *http.Client { + return &http.Client{ + Transport: g.Transport(), + CheckRedirect: g.CheckRedirect, + Timeout: 20 * time.Second, + } +} diff --git a/internal/guard/guard_test.go b/internal/guard/guard_test.go new file mode 100644 index 0000000..ad2a647 --- /dev/null +++ b/internal/guard/guard_test.go @@ -0,0 +1,541 @@ +package guard + +import ( + "context" + "errors" + "net" + "net/http" + "net/netip" + "net/url" + "strings" + "syscall" + "testing" + "time" +) + +type nopConn struct{} + +func (nopConn) Control(func(uintptr)) error { return nil } +func (nopConn) Read(func(uintptr) bool) error { return nil } +func (nopConn) Write(func(uintptr) bool) error { return nil } + +func mustAddr(s string) netip.Addr { + a, err := netip.ParseAddr(s) + if err != nil { + panic(err) + } + return a +} + +func TestCheckAddrTable(t *testing.T) { + t.Parallel() + tests := []struct { + addr string + blocked bool + reason string + }{ + {"8.8.8.8", false, ""}, + {"1.1.1.1", false, ""}, + {"93.184.216.34", false, ""}, + {"2001:4860:4860::8888", false, ""}, + {"172.15.255.255", false, ""}, // just below 172.16/12 + {"172.32.0.1", false, ""}, // just above 172.16/12 + {"100.63.255.255", false, ""}, // just below CGNAT + {"100.128.0.1", false, ""}, // just above CGNAT + + {"127.0.0.1", true, "loopback"}, + {"127.255.255.254", true, "loopback"}, + {"::1", true, "loopback"}, + {"10.0.0.5", true, "private"}, + {"10.255.255.255", true, "private"}, + {"192.168.1.1", true, "private"}, + {"172.16.0.1", true, "private"}, + {"172.31.255.255", true, "private"}, + {"100.64.0.1", true, "tailnet"}, + {"100.95.252.120", true, "tailnet"}, // rift + {"100.127.255.254", true, "tailnet"}, + {"fd7a:115c:a1e0::1", true, "tailnet"}, + {"fd7a:115c:a1e0:ffff::1", true, "tailnet"}, + {"fc00::1", true, "private"}, // ULA + {"fe80::1", true, "link-local"}, + {"169.254.169.254", true, "link-local"}, // cloud metadata + {"224.0.0.1", true, "multicast"}, + {"ff02::1", true, "multicast"}, + {"0.0.0.0", true, "unspecified"}, + {"0.1.2.3", true, "unspecified"}, // 0.0.0.0/8 + {"::", true, "unspecified"}, + {"::ffff:127.0.0.1", true, "loopback"}, + {"::ffff:10.0.0.1", true, "private"}, + {"::ffff:c0a8:1", true, "private"}, // 192.168.0.1 mapped + {"::ffff:8.8.8.8", false, ""}, // mapped public → unmapped + {"64:ff9b::10.0.0.1", true, "NAT64"}, + {"64:ff9b::8.8.8.8", true, "NAT64"}, + {"64:ff9b::a00:1", true, "NAT64"}, // 10.0.0.1 + } + for _, tt := range tests { + t.Run(tt.addr, func(t *testing.T) { + t.Parallel() + err := CheckAddr(mustAddr(tt.addr)) + if tt.blocked && err == nil { + t.Fatalf("CheckAddr(%s) = nil, want blocked (%s)", tt.addr, tt.reason) + } + if !tt.blocked && err != nil { + t.Fatalf("CheckAddr(%s) = %v, want allow", tt.addr, err) + } + if tt.blocked && err != nil && tt.reason != "" && !strings.Contains(err.Error(), tt.reason) && !strings.Contains(err.Error(), "private") && !strings.Contains(err.Error(), "loopback") && !strings.Contains(err.Error(), "tailnet") && !strings.Contains(err.Error(), "NAT64") && !strings.Contains(err.Error(), "unspecified") && !strings.Contains(err.Error(), "link-local") && !strings.Contains(err.Error(), "multicast") && !strings.Contains(err.Error(), "non-global") { + t.Logf("CheckAddr(%s) = %v (wanted reason %q; format is still blocked:)", tt.addr, err, tt.reason) + } + if tt.blocked && err != nil && !strings.HasPrefix(err.Error(), "blocked:") { + t.Fatalf("error %q does not start with blocked:", err) + } + }) + } +} + +func TestParseURLRejects(t *testing.T) { + t.Parallel() + cases := []struct { + raw string + substr string + }{ + {"", "empty"}, + {"ftp://example.com/", "scheme"}, + {"file:///etc/passwd", "scheme"}, + {"gopher://example.com/", "scheme"}, + {"http://user:pass@example.com/", "userinfo"}, + {"http://user@example.com/", "userinfo"}, + {"http://127.0.0.1:80@8.8.8.8/", "userinfo"}, + {"http:///", "missing host"}, + {"http://example.com:8080/", "port"}, + {"http://example.com:22/", "port"}, + {"http://dixie:11434", "port"}, + {"http://100.95.252.120:8000", "port"}, + {"https://example.com:8443/", "port"}, + {"http://exa mple.com/", "invalid url"}, + } + for _, tt := range cases { + t.Run(tt.raw, func(t *testing.T) { + _, _, err := ParseURL(tt.raw) + if err == nil { + t.Fatalf("ParseURL(%q) = nil, want error", tt.raw) + } + if tt.substr != "" && !strings.Contains(err.Error(), tt.substr) { + t.Fatalf("ParseURL(%q) = %v, want substr %q", tt.raw, err, tt.substr) + } + }) + } +} + +func TestParseURLAllows80And443(t *testing.T) { + t.Parallel() + for _, raw := range []string{ + "http://example.com/", + "http://example.com:80/", + "https://example.com/", + "https://example.com:443/", + "http://example.com:443/", + "https://example.com:80/", + } { + if _, _, err := ParseURL(raw); err != nil { + t.Errorf("ParseURL(%q) = %v, want nil", raw, err) + } + } +} + +func TestDecimalOctalHexIPs(t *testing.T) { + t.Parallel() + // All of these are 127.0.0.1 in inet_aton form. + loopbacks := []string{ + "http://127.0.0.1/", + "http://2130706433/", // decimal + "http://0x7f000001/", // hex dword + "http://017700000001/", // octal dword + "http://0177.0.0.1/", // octal dotted + "http://0x7f.0.0.1/", // hex dotted + "http://127.1/", // short + "http://127.0.1/", // 3-part + "http://0x7f.1/", + } + g := &Guard{} + ctx := context.Background() + for _, raw := range loopbacks { + t.Run(raw, func(t *testing.T) { + _, _, err := g.CheckURL(ctx, raw) + if err == nil { + t.Fatalf("CheckURL(%q) allowed, want blocked", raw) + } + if !strings.HasPrefix(err.Error(), "blocked:") { + t.Fatalf("CheckURL(%q) = %v, want blocked:", raw, err) + } + // These must be classified as IP literals, never sent to DNS. + }) + } + + privates := []string{ + "http://10.0.0.1/", + "http://0x0a000001/", + "http://167772161/", // 10.0.0.1 decimal + "http://192.168.0.1/", + "http://0xc0a80001/", + } + for _, raw := range privates { + t.Run(raw, func(t *testing.T) { + _, _, err := g.CheckURL(ctx, raw) + if err == nil { + t.Fatalf("CheckURL(%q) allowed, want blocked", raw) + } + }) + } +} + +func TestNamedCasesFromPlan(t *testing.T) { + t.Parallel() + g := &Guard{ + LookupIP: func(_ context.Context, host string) ([]netip.Addr, error) { + switch host { + case "dixie": + return []netip.Addr{mustAddr("100.64.0.10")}, nil + default: + t.Fatalf("unexpected lookup %q", host) + return nil, nil + } + }, + } + ctx := context.Background() + cases := []string{ + "http://100.95.252.120:8000", + "http://dixie:11434", + "http://[fd7a:115c:a1e0::1]", + "http://[fd7a:115c:a1e0::1]/", + } + for _, raw := range cases { + t.Run(raw, func(t *testing.T) { + _, _, err := g.CheckURL(ctx, raw) + if err == nil { + t.Fatalf("CheckURL(%q) allowed", raw) + } + var b *BlockedError + if !errors.As(err, &b) { + t.Fatalf("CheckURL(%q) = %v (%T), want BlockedError", raw, err, err) + } + }) + } +} + +func TestMixedRecordsRejected(t *testing.T) { + t.Parallel() + g := &Guard{ + LookupIP: func(_ context.Context, host string) ([]netip.Addr, error) { + return []netip.Addr{mustAddr("8.8.8.8"), mustAddr("10.0.0.1")}, nil + }, + } + _, _, err := g.CheckURL(context.Background(), "http://evil.example/") + if err == nil { + t.Fatal("mixed public+private records allowed") + } + if !strings.Contains(err.Error(), "10.0.0.1") { + t.Fatalf("error %v should name the private address", err) + } +} + +func TestAllPublicRecordsAllowed(t *testing.T) { + t.Parallel() + g := &Guard{ + LookupIP: func(_ context.Context, host string) ([]netip.Addr, error) { + return []netip.Addr{mustAddr("8.8.8.8"), mustAddr("2001:4860:4860::8888")}, nil + }, + } + _, addrs, err := g.CheckURL(context.Background(), "https://dns.google/") + if err != nil { + t.Fatal(err) + } + if len(addrs) != 2 { + t.Fatalf("got %d addrs", len(addrs)) + } +} + +func TestRebindingBlockedAtControl(t *testing.T) { + t.Parallel() + g := &Guard{ + LookupIP: func(_ context.Context, host string) ([]netip.Addr, error) { + // Pre-check sees a public address. + return []netip.Addr{mustAddr("93.184.216.34")}, nil + }, + } + _, _, err := g.CheckURL(context.Background(), "http://rebind.example/") + if err != nil { + t.Fatalf("pre-check should pass on public record: %v", err) + } + // After pre-check, DNS rebinds to loopback. Control sees the dial IP. + err = g.Control("tcp", "127.0.0.1:80", nopConn{}) + if err == nil { + t.Fatal("Control allowed 127.0.0.1 after rebind") + } + if !strings.Contains(err.Error(), "127.0.0.1") { + t.Fatalf("Control error %v should name 127.0.0.1", err) + } + err = g.Control("tcp", "10.0.0.5:443", nopConn{}) + if err == nil { + t.Fatal("Control allowed 10.0.0.5") + } + err = g.Control("tcp", "8.8.8.8:80", nopConn{}) + if err != nil { + t.Fatalf("Control public: %v", err) + } + err = g.Control("tcp", "8.8.8.8:22", nopConn{}) + if err == nil { + t.Fatal("Control allowed port 22") + } +} + +func TestControlHappyEyeballsPrivateLeg(t *testing.T) { + t.Parallel() + g := &Guard{} + // Simulate two racing dials; the private one must fail even if the + // public one would succeed. + if err := g.Control("tcp", "1.1.1.1:443", nopConn{}); err != nil { + t.Fatalf("public leg: %v", err) + } + if err := g.Control("tcp6", "[::1]:443", nopConn{}); err == nil { + t.Fatal("loopback v6 leg allowed") + } + if err := g.Control("tcp", "100.95.252.120:80", nopConn{}); err == nil { + t.Fatal("tailnet leg allowed") + } +} + +func TestRedirectToPrivate(t *testing.T) { + t.Parallel() + g := &Guard{ + LookupIP: func(_ context.Context, host string) ([]netip.Addr, error) { + return []netip.Addr{mustAddr("93.184.216.34")}, nil + }, + } + orig, err := http.NewRequest("GET", "http://example.com/", nil) + if err != nil { + t.Fatal(err) + } + next, err := http.NewRequest("GET", "http://127.0.0.1/secret", nil) + if err != nil { + t.Fatal(err) + } + if err := g.CheckRedirect(next, []*http.Request{orig}); err == nil { + t.Fatal("redirect to 127.0.0.1 allowed") + } + + next, _ = http.NewRequest("GET", "http://10.0.0.5/", nil) + if err := g.CheckRedirect(next, []*http.Request{orig}); err == nil { + t.Fatal("redirect to 10.0.0.5 allowed") + } + + next, _ = http.NewRequest("GET", "http://[fd7a:115c:a1e0::1]/", nil) + if err := g.CheckRedirect(next, []*http.Request{orig}); err == nil { + t.Fatal("redirect to tailnet ULA allowed") + } + + next, _ = http.NewRequest("GET", "http://dixie:11434/", nil) + if err := g.CheckRedirect(next, []*http.Request{orig}); err == nil { + t.Fatal("redirect to dixie:11434 allowed") + } + + next, _ = http.NewRequest("GET", "https://safe.example/", nil) + if err := g.CheckRedirect(next, []*http.Request{orig}); err != nil { + t.Fatalf("redirect to public https: %v", err) + } +} + +func TestTooManyRedirects(t *testing.T) { + t.Parallel() + g := &Guard{ + LookupIP: func(_ context.Context, _ string) ([]netip.Addr, error) { + return []netip.Addr{mustAddr("93.184.216.34")}, nil + }, + } + via := make([]*http.Request, 5) + for i := range via { + via[i], _ = http.NewRequest("GET", "http://example.com/", nil) + } + next, _ := http.NewRequest("GET", "http://example.com/next", nil) + err := g.CheckRedirect(next, via) + if err == nil || !strings.Contains(err.Error(), "too many redirects") { + t.Fatalf("got %v", err) + } +} + +func TestDialLoopbackIsBlocked(t *testing.T) { + t.Parallel() + g := &Guard{} + d := g.Dialer() + c, err := d.Dial("tcp", "127.0.0.1:80") + if err == nil { + c.Close() + t.Fatal("Dial 127.0.0.1:80 succeeded") + } + if !strings.HasPrefix(err.Error(), "blocked:") && !strings.Contains(err.Error(), "blocked:") { + t.Fatalf("Dial error %v should be a blocked: error", err) + } + c, err = d.Dial("tcp", "[::1]:443") + if err == nil { + c.Close() + t.Fatal("Dial [::1]:443 succeeded") + } +} + +func TestTransportHasNoProxy(t *testing.T) { + t.Parallel() + g := &Guard{} + tr := g.Transport() + if tr.Proxy != nil { + req, _ := http.NewRequest("GET", "http://example.com/", nil) + if u, err := tr.Proxy(req); u != nil || err != nil { + t.Fatalf("Proxy returned %v, %v; want nil", u, err) + } + } +} + +func TestBlockedErrorFormat(t *testing.T) { + t.Parallel() + err := CheckAddr(mustAddr("10.0.0.5")) + if err == nil || err.Error() != "blocked: private address 10.0.0.5" { + t.Fatalf("got %v", err) + } +} + +func TestNoAddresses(t *testing.T) { + t.Parallel() + g := &Guard{ + LookupIP: func(_ context.Context, _ string) ([]netip.Addr, error) { + return nil, nil + }, + } + _, _, err := g.CheckURL(context.Background(), "http://empty.example/") + if err == nil || !strings.Contains(err.Error(), "no addresses") { + t.Fatalf("got %v", err) + } +} + +func TestParseIPv4AtOn(t *testing.T) { + t.Parallel() + got, ok := parseIPv4AtOn("2130706433") + if !ok || got.String() != "127.0.0.1" { + t.Fatalf("decimal: got %v ok=%v", got, ok) + } + got, ok = parseIPv4AtOn("0177.0.0.1") + if !ok || got.String() != "127.0.0.1" { + t.Fatalf("octal dotted: got %v ok=%v", got, ok) + } + got, ok = parseIPv4AtOn("0x7f.0.0.1") + if !ok || got.String() != "127.0.0.1" { + t.Fatalf("hex dotted: got %v ok=%v", got, ok) + } + got, ok = parseIPv4AtOn("127.1") + if !ok || got.String() != "127.0.0.1" { + t.Fatalf("short: got %v ok=%v", got, ok) + } + if _, ok := parseIPv4AtOn("example.com"); ok { + t.Fatal("hostname parsed as IP") + } + if _, ok := parseIPv4AtOn("256.1.1.1"); ok { + t.Fatal("256.1.1.1 should not parse") + } +} + +func TestClientCheckRedirectWired(t *testing.T) { + t.Parallel() + g := &Guard{} + cl := g.Client() + if cl.CheckRedirect == nil { + t.Fatal("CheckRedirect not set") + } + if cl.Timeout != 20*time.Second { + t.Fatalf("Timeout = %v", cl.Timeout) + } +} + +func TestURLStringRoundTripRedirect(t *testing.T) { + t.Parallel() + // A 302 Location that url.URL.String() produces must still be rejected. + u, err := url.Parse("http://127.0.0.1/x") + if err != nil { + t.Fatal(err) + } + g := &Guard{} + req := &http.Request{URL: u} + via := []*http.Request{{URL: mustURL("http://example.com/")}} + if err := g.CheckRedirect(req, via); err == nil { + t.Fatal("allowed") + } +} + +func mustURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + return u +} + +// Ensure Control implements the Dialer hook shape used by net.Dialer. +var ( + _ func(network, address string, c syscall.RawConn) error = (*Guard)(nil).Control + _ func(ctx context.Context, network, address string, c syscall.RawConn) error = (*Guard)(nil).ControlContext + _ func(req *http.Request, via []*http.Request) error = (*Guard)(nil).CheckRedirect +) + +func TestLocalhostName(t *testing.T) { + t.Parallel() + g := &Guard{ + LookupIP: func(_ context.Context, host string) ([]netip.Addr, error) { + if host != "localhost" { + t.Fatalf("host %q", host) + } + return []netip.Addr{mustAddr("127.0.0.1"), mustAddr("::1")}, nil + }, + } + _, _, err := g.CheckURL(context.Background(), "http://localhost/") + if err == nil { + t.Fatal("localhost allowed") + } +} + +func TestNonASCIIHostRejected(t *testing.T) { + t.Parallel() + _, _, err := ParseURL("http://exämple.com/") + if err == nil { + t.Fatal("non-ascii host allowed") + } +} + +func TestPublicIPLiteralAllowed(t *testing.T) { + t.Parallel() + g := &Guard{} + _, addrs, err := g.CheckURL(context.Background(), "http://1.1.1.1/") + if err != nil { + t.Fatal(err) + } + if len(addrs) != 1 || addrs[0].String() != "1.1.1.1" { + t.Fatalf("got %v", addrs) + } +} + +func TestSplitHostPortIPv6Control(t *testing.T) { + t.Parallel() + g := &Guard{} + if err := g.Control("tcp", "[fd7a:115c:a1e0::1]:80", nopConn{}); err == nil { + t.Fatal("allowed tailnet v6") + } + if err := g.Control("tcp", "[2001:4860:4860::8888]:443", nopConn{}); err != nil { + t.Fatal(err) + } +} + +func TestDialerType(t *testing.T) { + t.Parallel() + d := (&Guard{}).Dialer() + if d.Control == nil || d.ControlContext == nil { + t.Fatal("missing Control hooks") + } + if _, ok := interface{}(d).(*net.Dialer); !ok { + t.Fatal("not a Dialer") + } +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..c3179bd --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,41 @@ +package server + +import ( + "encoding/json" + "net/http" + "time" +) + +// Version is set from main via New. +type Server struct { + Version string + Mux *http.ServeMux +} + +func New(version string) *Server { + s := &Server{Version: version, Mux: http.NewServeMux()} + s.Mux.HandleFunc("GET /healthz", s.healthz) + s.Mux.HandleFunc("GET /healthz/", s.healthz) + return s +} + +func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]any{ + "status": "ok", + "version": s.Version, + }) +} + +// HTTPServer wraps the mux with timeouts. Listen address is the caller's. +func (s *Server) HTTPServer(addr string) *http.Server { + return &http.Server{ + Addr: addr, + Handler: s.Mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 30 * time.Second, + } +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..96a9f04 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,29 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestHealthz(t *testing.T) { + t.Parallel() + s := New("testver") + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + rec := httptest.NewRecorder() + s.Mux.ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("status %d", rec.Code) + } + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body["status"] != "ok" { + t.Fatalf("status=%v", body["status"]) + } + if body["version"] != "testver" { + t.Fatalf("version=%v", body["version"]) + } +} diff --git a/nix/module.nix b/nix/module.nix new file mode 100644 index 0000000..646c8ee --- /dev/null +++ b/nix/module.nix @@ -0,0 +1,150 @@ +{ config, lib, pkgs, ... }: +let + cfg = config.services.sift; + settingsFormat = pkgs.formats.toml { }; +in +{ + options.services.sift = { + enable = lib.mkEnableOption "sift LLM-oriented fetch and search"; + + package = lib.mkOption { + type = lib.types.package; + description = "sift package. The flake module sets this to packages.sift."; + }; + + listen = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1:8890"; + description = "Address the HTTP server binds to (loopback by default)."; + }; + + tailscaleServe = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Advertise the listener on tailscale serve at https://:8443."; + }; + + tailscaleHttpsPort = lib.mkOption { + type = lib.types.port; + default = 8443; + description = "HTTPS port for tailscale serve (443 is SearXNG on orion)."; + }; + + settings = lib.mkOption { + type = settingsFormat.type; + default = { }; + description = "Written to /etc/sift/config.toml. Unused keys are reserved for P1+."; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.package != null; + message = "services.sift.package must be set (the flake module does this)."; + } + ]; + + users.users.sift = { + isSystemUser = true; + group = "sift"; + home = "/var/lib/sift"; + }; + users.groups.sift = { }; + + environment.etc."sift/config.toml".source = settingsFormat.generate "sift.toml" ( + { listen = cfg.listen; } // cfg.settings + ); + + # Keys and env secrets are created on-box, never in the store. + systemd.tmpfiles.rules = [ + "d /var/lib/sift 0750 root sift -" + ]; + + system.activationScripts.sift-secrets = { + deps = [ "users" ]; + text = '' + install -d -m 0750 -o root -g sift /var/lib/sift + if [ ! -f /var/lib/sift/keys.toml ]; then + umask 027 + printf '%s\n' '# key-hash → { name, workspace }. Generated empty; fill on-box.' > /var/lib/sift/keys.toml + chown root:sift /var/lib/sift/keys.toml + chmod 0640 /var/lib/sift/keys.toml + fi + if [ ! -f /var/lib/sift/secret.env ]; then + umask 027 + printf '%s\n' '# BRAVE_API_KEY= (set on-box; do not read SearXNG secret.env)' > /var/lib/sift/secret.env + chown root:sift /var/lib/sift/secret.env + chmod 0640 /var/lib/sift/secret.env + fi + ''; + }; + + systemd.services.sift = { + description = "sift LLM-oriented fetch and search"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "simple"; + User = "sift"; + Group = "sift"; + ExecStart = "${lib.getExe cfg.package} -listen ${cfg.listen} -config /etc/sift/config.toml"; + WorkingDirectory = "/var/lib/sift"; + Restart = "on-failure"; + RestartSec = "5s"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + ProtectHostname = true; + ProtectClock = true; + RestrictAddressFamilies = [ + "AF_UNIX" + "AF_INET" + "AF_INET6" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + ]; + CapabilityBoundingSet = ""; + AmbientCapabilities = ""; + ReadWritePaths = [ "/var/lib/sift" ]; + ConfigurationDirectory = "sift"; + StateDirectory = "sift"; + }; + }; + + systemd.services.tailscale-serve-sift = lib.mkIf cfg.tailscaleServe { + description = "Advertise sift on Tailscale Serve :${toString cfg.tailscaleHttpsPort}"; + after = [ + "tailscaled.service" + "network-online.target" + "sift.service" + ]; + wants = [ + "tailscaled.service" + "network-online.target" + ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + Restart = "on-failure"; + RestartSec = "5s"; + ExecStart = "${lib.getExe pkgs.tailscale} serve --bg --yes --https=${toString cfg.tailscaleHttpsPort} http://${cfg.listen}"; + }; + }; + }; +}