diff --git a/cmd/sift/serve.go b/cmd/sift/serve.go
index cb9a145..6ae956b 100644
--- a/cmd/sift/serve.go
+++ b/cmd/sift/serve.go
@@ -106,13 +106,14 @@ func serveMain() {
log.Warn("digest cache", "err", err)
}
srv.Digest = &digest.Engine{
- ChatURL: cfg.Dixie.Chat,
- EmbedURL: cfg.Dixie.Embed,
- Model: cfg.Digest.Model,
- Thinking: cfg.Digest.Thinking,
- Gate: digest.NewGate(cfg.Dixie.MaxInflight, cfg.Dixie.QueueDepth, config.ParseDuration(cfg.Dixie.QueueWait, digest.DefaultQWait)),
- Cache: cache,
- Log: log,
+ ChatURL: cfg.Dixie.Chat,
+ EmbedURL: cfg.Dixie.Embed,
+ EmbedModel: cfg.Dixie.EmbedModel,
+ Model: cfg.Digest.Model,
+ Thinking: cfg.Digest.Thinking,
+ Gate: digest.NewGate(cfg.Dixie.MaxInflight, cfg.Dixie.QueueDepth, config.ParseDuration(cfg.Dixie.QueueWait, digest.DefaultQWait)),
+ Cache: cache,
+ Log: log,
}
}
if cfg.Context.Honcho != "" {
diff --git a/internal/api/api.go b/internal/api/api.go
index ab3fcf1..2c741e9 100644
--- a/internal/api/api.go
+++ b/internal/api/api.go
@@ -302,6 +302,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if s.Digest == nil {
warn = append(warn, "digest fell back to clean: unset")
mode = "clean"
+ content, warn = capDigestFallback(content, warn)
digMeta = map[string]any{"fallback": "unset"}
} else {
dr := s.Digest.Run(r.Context(), content, q, res.FinalURL)
@@ -309,7 +310,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if dr.Fallback != "" {
warn = append(warn, "digest fell back to clean: "+dr.Fallback)
mode = "clean"
- content = clean.TruncateUTF8(content, s.OutCap)
+ content, warn = capDigestFallback(content, warn)
digMeta = map[string]any{"fallback": dr.Fallback, "model": dr.Model}
} else {
content = dr.Text
@@ -552,6 +553,13 @@ func (s *Server) release(caller, rawURL string) {
}
}
+func capDigestFallback(content string, warn []string) (string, []string) {
+ if digest.EstTokens(content) <= digest.SmallDoc {
+ return content, warn
+ }
+ return digest.CapBudget(content), append(warn, "clean fallback truncated to digest budget")
+}
+
func hostOf(raw string) string {
u, err := url.Parse(raw)
if err != nil {
diff --git a/internal/api/api_test.go b/internal/api/api_test.go
index abb0170..a0d6b73 100644
--- a/internal/api/api_test.go
+++ b/internal/api/api_test.go
@@ -16,6 +16,7 @@ import (
"time"
"git.wntrmute.dev/kyle/sift/internal/auth"
+ "git.wntrmute.dev/kyle/sift/internal/digest"
"git.wntrmute.dev/kyle/sift/internal/fetch"
"git.wntrmute.dev/kyle/sift/internal/guard"
)
@@ -156,6 +157,18 @@ func TestDigestFallbackWhenUnset(t *testing.T) {
}
}
+func TestCapDigestFallback(t *testing.T) {
+ big := strings.Repeat("word ", 50000)
+ out, warn := capDigestFallback(big, nil)
+ if digest.EstTokens(out) > digest.SmallDoc {
+ t.Fatalf("tokens %d", digest.EstTokens(out))
+ }
+ joined := strings.Join(warn, " ")
+ if !strings.Contains(joined, "digest budget") {
+ t.Fatalf("warn %v", warn)
+ }
+}
+
func TestNever502(t *testing.T) {
s, key := setupAPI(t, nil)
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})
diff --git a/internal/auth/auth.go b/internal/auth/auth.go
index 89952f7..3ad9a99 100644
--- a/internal/auth/auth.go
+++ b/internal/auth/auth.go
@@ -8,8 +8,10 @@ import (
"encoding/hex"
"fmt"
"os"
+ "path/filepath"
"strings"
"sync"
+ "syscall"
"github.com/pelletier/go-toml/v2"
)
@@ -112,11 +114,7 @@ func Add(path, name, workspace, keyOut string) error {
if err != nil {
return err
}
- tmp := path + ".tmp"
- if err := os.WriteFile(tmp, out, 0o640); err != nil {
- return err
- }
- return os.Rename(tmp, path)
+ return writePreserving(path, out)
}
func List(path string) ([]Caller, error) {
@@ -151,5 +149,49 @@ func Revoke(path, name string) error {
if err != nil {
return err
}
- return os.WriteFile(path, raw, 0o640)
+ return writePreserving(path, raw)
+}
+
+func writePreserving(path string, data []byte) error {
+ mode := os.FileMode(0o640)
+ uid, gid := -1, -1
+ if st, err := os.Stat(path); err == nil {
+ mode = st.Mode()
+ if sys, ok := st.Sys().(*syscall.Stat_t); ok {
+ uid, gid = int(sys.Uid), int(sys.Gid)
+ }
+ }
+ f, err := os.CreateTemp(filepath.Dir(path), ".sift-keys-*.tmp")
+ if err != nil {
+ return err
+ }
+ tmp := f.Name()
+ ok := false
+ defer func() {
+ if !ok {
+ _ = os.Remove(tmp)
+ }
+ }()
+ if _, err := f.Write(data); err != nil {
+ _ = f.Close()
+ return err
+ }
+ if err := f.Chmod(mode.Perm()); err != nil {
+ _ = f.Close()
+ return err
+ }
+ if uid >= 0 {
+ if err := f.Chown(uid, gid); err != nil {
+ _ = f.Close()
+ return err
+ }
+ }
+ if err := f.Close(); err != nil {
+ return err
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ return err
+ }
+ ok = true
+ return nil
}
diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go
index cd4a037..5585b4d 100644
--- a/internal/auth/auth_test.go
+++ b/internal/auth/auth_test.go
@@ -50,3 +50,22 @@ func TestAddLookupRevoke(t *testing.T) {
t.Fatal("revoked still valid")
}
}
+
+func TestAddPreservesKeysMode(t *testing.T) {
+ dir := t.TempDir()
+ keys := filepath.Join(dir, "keys.toml")
+ if err := os.WriteFile(keys, []byte(""), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ out := filepath.Join(dir, "k")
+ if err := Add(keys, "talos", "hermes-fleet", out); err != nil {
+ t.Fatal(err)
+ }
+ st, err := os.Stat(keys)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if st.Mode().Perm() != 0o600 {
+ t.Fatalf("mode %o", st.Mode().Perm())
+ }
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index cf5c21e..60bed4d 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -21,6 +21,7 @@ type File struct {
type Dixie struct {
Chat string `toml:"chat"`
Embed string `toml:"embed"`
+ EmbedModel string `toml:"embed_model"`
MaxInflight int `toml:"max_inflight"`
QueueDepth int `toml:"queue_depth"`
QueueWait string `toml:"queue_wait"`
diff --git a/internal/digest/digest.go b/internal/digest/digest.go
index b5d2676..ff14fec 100644
--- a/internal/digest/digest.go
+++ b/internal/digest/digest.go
@@ -154,15 +154,16 @@ func (g *Gate) Release() {
}
type Engine struct {
- ChatURL string
- EmbedURL string
- Model string
- Thinking bool
- Client *http.Client
- Gate *Gate
- Timeout time.Duration
- Log *slog.Logger
- Cache *dcache.DB
+ ChatURL string
+ EmbedURL string
+ EmbedModel string
+ Model string
+ Thinking bool
+ Client *http.Client
+ Gate *Gate
+ Timeout time.Duration
+ Log *slog.Logger
+ Cache *dcache.DB
mu sync.Mutex
Calls uint64
@@ -201,6 +202,13 @@ func (e *Engine) model() string {
return "ornith-1.5-9b-uncensored"
}
+func (e *Engine) embedModel() string {
+ if e != nil && e.EmbedModel != "" {
+ return e.EmbedModel
+ }
+ return "honcho-embed"
+}
+
func (e *Engine) noteFallback(reason string) {
if e == nil {
return
@@ -379,7 +387,7 @@ func (e *Engine) embed(ctx context.Context, inputs []string) ([][]float64, error
}
func (e *Engine) embedBatch(ctx context.Context, inputs []string) ([][]float64, error) {
- payload, _ := json.Marshal(map[string]any{"input": inputs, "model": "qwen3-embedding-0.6b"})
+ payload, _ := json.Marshal(map[string]any{"input": inputs, "model": e.embedModel()})
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.EmbedURL, bytes.NewReader(payload))
if err != nil {
return nil, err
@@ -541,6 +549,9 @@ func countBudget(chunks []string, budget int) int {
return n
}
+// CapBudget trims s to the digest input budget (SmallDoc estimated tokens).
+func CapBudget(s string) string { return trimTokens(s, SmallDoc) }
+
func trimTokens(s string, max int) string {
r := []rune(s)
n := max * 4
diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go
index 3de0107..b4c592c 100644
--- a/internal/digest/digest_test.go
+++ b/internal/digest/digest_test.go
@@ -250,3 +250,42 @@ func TestChunkOrderAndBudget(t *testing.T) {
t.Fatal("document order")
}
}
+
+func TestEmbedUsesConfiguredModel(t *testing.T) {
+ var got string
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var body struct {
+ Model string `json:"model"`
+ }
+ _ = json.NewDecoder(r.Body).Decode(&body)
+ got = body.Model
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "data": []map[string]any{
+ {"index": 0, "embedding": []float64{1, 0}},
+ {"index": 1, "embedding": []float64{0, 1}},
+ },
+ })
+ }))
+ defer ts.Close()
+ e := &Engine{EmbedURL: ts.URL, EmbedModel: "honcho-embed", Client: ts.Client()}
+ if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
+ t.Fatal(err)
+ }
+ if got != "honcho-embed" {
+ t.Fatalf("model %q", got)
+ }
+ e.EmbedModel = "other-embed"
+ if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
+ t.Fatal(err)
+ }
+ if got != "other-embed" {
+ t.Fatalf("model %q", got)
+ }
+ e.EmbedModel = ""
+ if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
+ t.Fatal(err)
+ }
+ if got != "honcho-embed" {
+ t.Fatalf("default model %q", got)
+ }
+}
diff --git a/internal/recover/recover.go b/internal/recover/recover.go
index 40d9595..03a265a 100644
--- a/internal/recover/recover.go
+++ b/internal/recover/recover.go
@@ -6,9 +6,11 @@ import (
"context"
"encoding/json"
"fmt"
+ "net/http"
"net/url"
"regexp"
"strings"
+ "time"
"git.wntrmute.dev/kyle/sift/internal/fetch"
)
@@ -61,9 +63,10 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
var env struct {
ArchivedSnapshots struct {
Closest struct {
- Available bool `json:"available"`
- URL string `json:"url"`
- Timestamp string `json:"timestamp"`
+ Available bool `json:"available"`
+ URL string `json:"url"`
+ Timestamp string `json:"timestamp"`
+ Status json.RawMessage `json:"status"`
} `json:"closest"`
} `json:"archived_snapshots"`
}
@@ -74,11 +77,15 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
if !c.Available || c.URL == "" {
return nil, fmt.Errorf("wayback: unavailable")
}
+ if !waybackStatusOK(c.Status) {
+ return nil, fmt.Errorf("wayback: status %s", strings.TrimSpace(string(c.Status)))
+ }
u, err := url.Parse(c.URL)
- if err != nil || u.Scheme != "https" || strings.ToLower(u.Hostname()) != "web.archive.org" {
+ if err != nil || (u.Scheme != "https" && u.Scheme != "http") || strings.ToLower(u.Hostname()) != "web.archive.org" {
return nil, fmt.Errorf("wayback: bad snapshot url")
}
- snapURL := insertID(c.URL)
+ u.Scheme = "https"
+ snapURL := insertID(u.String())
page, err := f.Fetch(ctx, snapURL)
if err != nil {
return nil, err
@@ -170,7 +177,26 @@ func mementoDatetime(res *fetch.Resource) string {
if res == nil || res.Header == nil {
return ""
}
- return strings.TrimSpace(res.Header.Get("Memento-Datetime"))
+ raw := strings.TrimSpace(res.Header.Get("Memento-Datetime"))
+ if raw == "" {
+ return ""
+ }
+ t, err := http.ParseTime(raw)
+ if err != nil {
+ return ""
+ }
+ if t.After(time.Now().Add(24 * time.Hour)) {
+ return ""
+ }
+ return t.UTC().Format(time.RFC3339)
+}
+
+func waybackStatusOK(raw json.RawMessage) bool {
+ if len(bytesTrim(raw)) == 0 || string(raw) == "null" {
+ return true
+ }
+ s := strings.Trim(strings.TrimSpace(string(raw)), `"`)
+ return s == "200"
}
func hostOf(raw string) string {
diff --git a/internal/recover/recover_test.go b/internal/recover/recover_test.go
index f3e7de0..8d76fbb 100644
--- a/internal/recover/recover_test.go
+++ b/internal/recover/recover_test.go
@@ -3,6 +3,7 @@ package recover
import (
"context"
"encoding/json"
+ "net/http"
"strings"
"testing"
"time"
@@ -166,3 +167,60 @@ func TestRedirectToPrivateBlocked(t *testing.T) {
t.Fatalf("err %v", err)
}
}
+
+func TestWaybackHTTPRewrittenAnd204Skipped(t *testing.T) {
+ const fixture = `{"url":"https://www.ft.com/","archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://www.ft.com/","timestamp":"20240101120000","status":"204"}}}`
+ f := &fakeFetch{
+ body: map[string][]byte{
+ "https://archive.org/wayback/available": []byte(fixture),
+ "https://archive.ph/newest/": []byte("Saved" + strings.Repeat("ok ", 200) + ""),
+ },
+ ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
+ }
+ r, err := Try(context.Background(), f, "https://www.ft.com/")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if r.Via != ViaToday {
+ t.Fatalf("via %s hits %v", r.Via, f.hits)
+ }
+ for _, h := range f.hits {
+ if strings.Contains(h, "web.archive.org") {
+ t.Fatalf("fetched 204 snapshot: %v", f.hits)
+ }
+ }
+
+ page := []byte("Article" + strings.Repeat("paragraph ", 80) + "")
+ avail := []byte(`{"archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://example.com/x","timestamp":"20240101120000","status":"200"}}}`)
+ f2 := &fakeFetch{
+ body: map[string][]byte{
+ "https://archive.org/wayback/available": avail,
+ "https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
+ },
+ ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
+ }
+ r, err = Try(context.Background(), f2, "https://example.com/x")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if r.Via != ViaWayback {
+ t.Fatalf("via %s hits %v", r.Via, f2.hits)
+ }
+}
+
+func TestMementoDatetime(t *testing.T) {
+ h := make(http.Header)
+ h.Set("Memento-Datetime", "Wed, 01 Jan 2020 12:00:00 GMT")
+ got := mementoDatetime(&fetch.Resource{Header: h})
+ if got != "2020-01-01T12:00:00Z" {
+ t.Fatalf("got %q", got)
+ }
+ h.Set("Memento-Datetime", "Mon, 31 Dec 2029 00:00:00 GMT")
+ if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
+ t.Fatalf("future %q", got)
+ }
+ h.Set("Memento-Datetime", "not-a-date")
+ if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
+ t.Fatalf("garbage %q", got)
+ }
+}