p2.1: honcho-embed, wayback http/204, memento, keys owner

Embed requests send dixie.embed_model (default honcho-embed).
Wayback accepts http://web.archive.org, rewrites to https, skips
non-200 closest.status. Memento-Datetime is parsed as an HTTP date
and future/garbage values become empty snapshot_at. keys add writes
via temp+fchown/chmod/rename so root does not clobber keys.toml.
Digest fallback is capped at the 6k-token budget.

cmd/sift/serve.go

@@ -106,13 +106,14 @@ func serveMain() {
 			log.Warn("digest cache", "err", err)
 		}
 		srv.Digest = &digest.Engine{
-			ChatURL:  cfg.Dixie.Chat,
+			ChatURL:    cfg.Dixie.Chat,
-			EmbedURL: cfg.Dixie.Embed,
+			EmbedURL:   cfg.Dixie.Embed,
-			Model:    cfg.Digest.Model,
+			EmbedModel: cfg.Dixie.EmbedModel,
-			Thinking: cfg.Digest.Thinking,
+			Model:      cfg.Digest.Model,
-			Gate:     digest.NewGate(cfg.Dixie.MaxInflight, cfg.Dixie.QueueDepth, config.ParseDuration(cfg.Dixie.QueueWait, digest.DefaultQWait)),
+			Thinking:   cfg.Digest.Thinking,
-			Cache:    cache,
+			Gate:       digest.NewGate(cfg.Dixie.MaxInflight, cfg.Dixie.QueueDepth, config.ParseDuration(cfg.Dixie.QueueWait, digest.DefaultQWait)),
-			Log:      log,
+			Cache:      cache,
+			Log:        log,
 		}
 	}
 	if cfg.Context.Honcho != "" {

internal/api/api.go

@@ -302,6 +302,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
 		if s.Digest == nil {
 			warn = append(warn, "digest fell back to clean: unset")
 			mode = "clean"
+			content, warn = capDigestFallback(content, warn)
 			digMeta = map[string]any{"fallback": "unset"}
 		} else {
 			dr := s.Digest.Run(r.Context(), content, q, res.FinalURL)
@@ -309,7 +310,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
 			if dr.Fallback != "" {
 				warn = append(warn, "digest fell back to clean: "+dr.Fallback)
 				mode = "clean"
-				content = clean.TruncateUTF8(content, s.OutCap)
+				content, warn = capDigestFallback(content, warn)
 				digMeta = map[string]any{"fallback": dr.Fallback, "model": dr.Model}
 			} else {
 				content = dr.Text
@@ -552,6 +553,13 @@ func (s *Server) release(caller, rawURL string) {
 	}
 }
 
+func capDigestFallback(content string, warn []string) (string, []string) {
+	if digest.EstTokens(content) <= digest.SmallDoc {
+		return content, warn
+	}
+	return digest.CapBudget(content), append(warn, "clean fallback truncated to digest budget")
+}
+
 func hostOf(raw string) string {
 	u, err := url.Parse(raw)
 	if err != nil {

internal/api/api_test.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"git.wntrmute.dev/kyle/sift/internal/auth"
+	"git.wntrmute.dev/kyle/sift/internal/digest"
 	"git.wntrmute.dev/kyle/sift/internal/fetch"
 	"git.wntrmute.dev/kyle/sift/internal/guard"
 )
@@ -156,6 +157,18 @@ func TestDigestFallbackWhenUnset(t *testing.T) {
 	}
 }
 
+func TestCapDigestFallback(t *testing.T) {
+	big := strings.Repeat("word ", 50000)
+	out, warn := capDigestFallback(big, nil)
+	if digest.EstTokens(out) > digest.SmallDoc {
+		t.Fatalf("tokens %d", digest.EstTokens(out))
+	}
+	joined := strings.Join(warn, " ")
+	if !strings.Contains(joined, "digest budget") {
+		t.Fatalf("warn %v", warn)
+	}
+}
+
 func TestNever502(t *testing.T) {
 	s, key := setupAPI(t, nil)
 	rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})

internal/auth/auth.go

@@ -8,8 +8,10 @@ import (
 	"encoding/hex"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
+	"syscall"
 
 	"github.com/pelletier/go-toml/v2"
 )
@@ -112,11 +114,7 @@ func Add(path, name, workspace, keyOut string) error {
 	if err != nil {
 		return err
 	}
-	tmp := path + ".tmp"
+	return writePreserving(path, out)
-	if err := os.WriteFile(tmp, out, 0o640); err != nil {
-		return err
-	}
-	return os.Rename(tmp, path)
 }
 
 func List(path string) ([]Caller, error) {
@@ -151,5 +149,49 @@ func Revoke(path, name string) error {
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(path, raw, 0o640)
+	return writePreserving(path, raw)
+}
+
+func writePreserving(path string, data []byte) error {
+	mode := os.FileMode(0o640)
+	uid, gid := -1, -1
+	if st, err := os.Stat(path); err == nil {
+		mode = st.Mode()
+		if sys, ok := st.Sys().(*syscall.Stat_t); ok {
+			uid, gid = int(sys.Uid), int(sys.Gid)
+		}
+	}
+	f, err := os.CreateTemp(filepath.Dir(path), ".sift-keys-*.tmp")
+	if err != nil {
+		return err
+	}
+	tmp := f.Name()
+	ok := false
+	defer func() {
+		if !ok {
+			_ = os.Remove(tmp)
+		}
+	}()
+	if _, err := f.Write(data); err != nil {
+		_ = f.Close()
+		return err
+	}
+	if err := f.Chmod(mode.Perm()); err != nil {
+		_ = f.Close()
+		return err
+	}
+	if uid >= 0 {
+		if err := f.Chown(uid, gid); err != nil {
+			_ = f.Close()
+			return err
+		}
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		return err
+	}
+	ok = true
+	return nil
 }

internal/auth/auth_test.go

@@ -50,3 +50,22 @@ func TestAddLookupRevoke(t *testing.T) {
 		t.Fatal("revoked still valid")
 	}
 }
+
+func TestAddPreservesKeysMode(t *testing.T) {
+	dir := t.TempDir()
+	keys := filepath.Join(dir, "keys.toml")
+	if err := os.WriteFile(keys, []byte(""), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	out := filepath.Join(dir, "k")
+	if err := Add(keys, "talos", "hermes-fleet", out); err != nil {
+		t.Fatal(err)
+	}
+	st, err := os.Stat(keys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.Mode().Perm() != 0o600 {
+		t.Fatalf("mode %o", st.Mode().Perm())
+	}
+}

internal/config/config.go

@@ -21,6 +21,7 @@ type File struct {
 type Dixie struct {
 	Chat        string `toml:"chat"`
 	Embed       string `toml:"embed"`
+	EmbedModel  string `toml:"embed_model"`
 	MaxInflight int    `toml:"max_inflight"`
 	QueueDepth  int    `toml:"queue_depth"`
 	QueueWait   string `toml:"queue_wait"`

internal/digest/digest.go

@@ -154,15 +154,16 @@ func (g *Gate) Release() {
 }
 
 type Engine struct {
-	ChatURL  string
+	ChatURL    string
-	EmbedURL string
+	EmbedURL   string
-	Model    string
+	EmbedModel string
-	Thinking bool
+	Model      string
-	Client   *http.Client
+	Thinking   bool
-	Gate     *Gate
+	Client     *http.Client
-	Timeout  time.Duration
+	Gate       *Gate
-	Log      *slog.Logger
+	Timeout    time.Duration
-	Cache    *dcache.DB
+	Log        *slog.Logger
+	Cache      *dcache.DB
 
 	mu       sync.Mutex
 	Calls    uint64
@@ -201,6 +202,13 @@ func (e *Engine) model() string {
 	return "ornith-1.5-9b-uncensored"
 }
 
+func (e *Engine) embedModel() string {
+	if e != nil && e.EmbedModel != "" {
+		return e.EmbedModel
+	}
+	return "honcho-embed"
+}
+
 func (e *Engine) noteFallback(reason string) {
 	if e == nil {
 		return
@@ -379,7 +387,7 @@ func (e *Engine) embed(ctx context.Context, inputs []string) ([][]float64, error
 }
 
 func (e *Engine) embedBatch(ctx context.Context, inputs []string) ([][]float64, error) {
-	payload, _ := json.Marshal(map[string]any{"input": inputs, "model": "qwen3-embedding-0.6b"})
+	payload, _ := json.Marshal(map[string]any{"input": inputs, "model": e.embedModel()})
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.EmbedURL, bytes.NewReader(payload))
 	if err != nil {
 		return nil, err
@@ -541,6 +549,9 @@ func countBudget(chunks []string, budget int) int {
 	return n
 }
 
+// CapBudget trims s to the digest input budget (SmallDoc estimated tokens).
+func CapBudget(s string) string { return trimTokens(s, SmallDoc) }
+
 func trimTokens(s string, max int) string {
 	r := []rune(s)
 	n := max * 4

internal/digest/digest_test.go

@@ -250,3 +250,42 @@ func TestChunkOrderAndBudget(t *testing.T) {
 		t.Fatal("document order")
 	}
 }
+
+func TestEmbedUsesConfiguredModel(t *testing.T) {
+	var got string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body struct {
+			Model string `json:"model"`
+		}
+		_ = json.NewDecoder(r.Body).Decode(&body)
+		got = body.Model
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"data": []map[string]any{
+				{"index": 0, "embedding": []float64{1, 0}},
+				{"index": 1, "embedding": []float64{0, 1}},
+			},
+		})
+	}))
+	defer ts.Close()
+	e := &Engine{EmbedURL: ts.URL, EmbedModel: "honcho-embed", Client: ts.Client()}
+	if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
+		t.Fatal(err)
+	}
+	if got != "honcho-embed" {
+		t.Fatalf("model %q", got)
+	}
+	e.EmbedModel = "other-embed"
+	if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
+		t.Fatal(err)
+	}
+	if got != "other-embed" {
+		t.Fatalf("model %q", got)
+	}
+	e.EmbedModel = ""
+	if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
+		t.Fatal(err)
+	}
+	if got != "honcho-embed" {
+		t.Fatalf("default model %q", got)
+	}
+}

internal/recover/recover.go

@@ -6,9 +6,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
+	"time"
 
 	"git.wntrmute.dev/kyle/sift/internal/fetch"
 )
@@ -61,9 +63,10 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
 	var env struct {
 		ArchivedSnapshots struct {
 			Closest struct {
-				Available bool   `json:"available"`
+				Available bool            `json:"available"`
-				URL       string `json:"url"`
+				URL       string          `json:"url"`
-				Timestamp string `json:"timestamp"`
+				Timestamp string          `json:"timestamp"`
+				Status    json.RawMessage `json:"status"`
 			} `json:"closest"`
 		} `json:"archived_snapshots"`
 	}
@@ -74,11 +77,15 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
 	if !c.Available || c.URL == "" {
 		return nil, fmt.Errorf("wayback: unavailable")
 	}
+	if !waybackStatusOK(c.Status) {
+		return nil, fmt.Errorf("wayback: status %s", strings.TrimSpace(string(c.Status)))
+	}
 	u, err := url.Parse(c.URL)
-	if err != nil || u.Scheme != "https" || strings.ToLower(u.Hostname()) != "web.archive.org" {
+	if err != nil || (u.Scheme != "https" && u.Scheme != "http") || strings.ToLower(u.Hostname()) != "web.archive.org" {
 		return nil, fmt.Errorf("wayback: bad snapshot url")
 	}
-	snapURL := insertID(c.URL)
+	u.Scheme = "https"
+	snapURL := insertID(u.String())
 	page, err := f.Fetch(ctx, snapURL)
 	if err != nil {
 		return nil, err
@@ -170,7 +177,26 @@ func mementoDatetime(res *fetch.Resource) string {
 	if res == nil || res.Header == nil {
 		return ""
 	}
-	return strings.TrimSpace(res.Header.Get("Memento-Datetime"))
+	raw := strings.TrimSpace(res.Header.Get("Memento-Datetime"))
+	if raw == "" {
+		return ""
+	}
+	t, err := http.ParseTime(raw)
+	if err != nil {
+		return ""
+	}
+	if t.After(time.Now().Add(24 * time.Hour)) {
+		return ""
+	}
+	return t.UTC().Format(time.RFC3339)
+}
+
+func waybackStatusOK(raw json.RawMessage) bool {
+	if len(bytesTrim(raw)) == 0 || string(raw) == "null" {
+		return true
+	}
+	s := strings.Trim(strings.TrimSpace(string(raw)), `"`)
+	return s == "200"
 }
 
 func hostOf(raw string) string {

internal/recover/recover_test.go

@@ -3,6 +3,7 @@ package recover
 import (
 	"context"
 	"encoding/json"
+	"net/http"
 	"strings"
 	"testing"
 	"time"
@@ -166,3 +167,60 @@ func TestRedirectToPrivateBlocked(t *testing.T) {
 		t.Fatalf("err %v", err)
 	}
 }
+
+func TestWaybackHTTPRewrittenAnd204Skipped(t *testing.T) {
+	const fixture = `{"url":"https://www.ft.com/","archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://www.ft.com/","timestamp":"20240101120000","status":"204"}}}`
+	f := &fakeFetch{
+		body: map[string][]byte{
+			"https://archive.org/wayback/available": []byte(fixture),
+			"https://archive.ph/newest/":            []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>"),
+		},
+		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
+	}
+	r, err := Try(context.Background(), f, "https://www.ft.com/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.Via != ViaToday {
+		t.Fatalf("via %s hits %v", r.Via, f.hits)
+	}
+	for _, h := range f.hits {
+		if strings.Contains(h, "web.archive.org") {
+			t.Fatalf("fetched 204 snapshot: %v", f.hits)
+		}
+	}
+
+	page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
+	avail := []byte(`{"archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://example.com/x","timestamp":"20240101120000","status":"200"}}}`)
+	f2 := &fakeFetch{
+		body: map[string][]byte{
+			"https://archive.org/wayback/available":                               avail,
+			"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
+		},
+		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
+	}
+	r, err = Try(context.Background(), f2, "https://example.com/x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.Via != ViaWayback {
+		t.Fatalf("via %s hits %v", r.Via, f2.hits)
+	}
+}
+
+func TestMementoDatetime(t *testing.T) {
+	h := make(http.Header)
+	h.Set("Memento-Datetime", "Wed, 01 Jan 2020 12:00:00 GMT")
+	got := mementoDatetime(&fetch.Resource{Header: h})
+	if got != "2020-01-01T12:00:00Z" {
+		t.Fatalf("got %q", got)
+	}
+	h.Set("Memento-Datetime", "Mon, 31 Dec 2029 00:00:00 GMT")
+	if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
+		t.Fatalf("future %q", got)
+	}
+	h.Set("Memento-Datetime", "not-a-date")
+	if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
+		t.Fatalf("garbage %q", got)
+	}
+}