kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

p2.1: honcho-embed, wayback http/204, memento, keys owner

Browse Source 
Embed requests send dixie.embed_model (default honcho-embed).
Wayback accepts http://web.archive.org, rewrites to https, skips
non-200 closest.status. Memento-Datetime is parsed as an HTTP date
and future/garbage values become empty snapshot_at. keys add writes
via temp+fchown/chmod/rename so root does not clobber keys.toml.
Digest fallback is capped at the 6k-token budget.
This commit is contained in:
kyle
2026-09-23 02:41:46 -07:00
parent 2db001e0d7
commit 16a7d678d3
10 changed files with 248 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/sift/serve.go
+8 -7
View File
@@ -106,13 +106,14 @@ func serveMain() {
log.Warn("digest cache", "err", err)
}
srv.Digest = &digest.Engine{
ChatURL: cfg.Dixie.Chat,
EmbedURL: cfg.Dixie.Embed,
Model: cfg.Digest.Model,
Thinking: cfg.Digest.Thinking,
Gate: digest.NewGate(cfg.Dixie.MaxInflight, cfg.Dixie.QueueDepth, config.ParseDuration(cfg.Dixie.QueueWait, digest.DefaultQWait)),
Cache: cache,
Log: log,
ChatURL: cfg.Dixie.Chat,
EmbedURL: cfg.Dixie.Embed,
EmbedModel: cfg.Dixie.EmbedModel,
Model: cfg.Digest.Model,
Thinking: cfg.Digest.Thinking,
Gate: digest.NewGate(cfg.Dixie.MaxInflight, cfg.Dixie.QueueDepth, config.ParseDuration(cfg.Dixie.QueueWait, digest.DefaultQWait)),
Cache: cache,
Log: log,
}
}
if cfg.Context.Honcho != "" {
internal/api/api.go
+9 -1
View File
@@ -302,6 +302,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if s.Digest == nil {
warn = append(warn, "digest fell back to clean: unset")
mode = "clean"
content, warn = capDigestFallback(content, warn)
digMeta = map[string]any{"fallback": "unset"}
} else {
dr := s.Digest.Run(r.Context(), content, q, res.FinalURL)
@@ -309,7 +310,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if dr.Fallback != "" {
warn = append(warn, "digest fell back to clean: "+dr.Fallback)
mode = "clean"
content = clean.TruncateUTF8(content, s.OutCap)
content, warn = capDigestFallback(content, warn)
digMeta = map[string]any{"fallback": dr.Fallback, "model": dr.Model}
} else {
content = dr.Text
@@ -552,6 +553,13 @@ func (s *Server) release(caller, rawURL string) {
}
}
func capDigestFallback(content string, warn []string) (string, []string) {
if digest.EstTokens(content) <= digest.SmallDoc {
return content, warn
}
return digest.CapBudget(content), append(warn, "clean fallback truncated to digest budget")
}
func hostOf(raw string) string {
u, err := url.Parse(raw)
if err != nil {
internal/api/api_test.go
+13
View File
@@ -16,6 +16,7 @@ import (
"time"
"git.wntrmute.dev/kyle/sift/internal/auth"
"git.wntrmute.dev/kyle/sift/internal/digest"
"git.wntrmute.dev/kyle/sift/internal/fetch"
"git.wntrmute.dev/kyle/sift/internal/guard"
)
@@ -156,6 +157,18 @@ func TestDigestFallbackWhenUnset(t *testing.T) {
}
}
func TestCapDigestFallback(t *testing.T) {
big := strings.Repeat("word ", 50000)
out, warn := capDigestFallback(big, nil)
if digest.EstTokens(out) > digest.SmallDoc {
t.Fatalf("tokens %d", digest.EstTokens(out))
}
joined := strings.Join(warn, " ")
if !strings.Contains(joined, "digest budget") {
t.Fatalf("warn %v", warn)
}
}
func TestNever502(t *testing.T) {
s, key := setupAPI(t, nil)
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})
internal/auth/auth.go
+48 -6
View File
@@ -8,8 +8,10 @@ import (
"encoding/hex"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"syscall"
"github.com/pelletier/go-toml/v2"
)
@@ -112,11 +114,7 @@ func Add(path, name, workspace, keyOut string) error {
if err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, out, 0o640); err != nil {
return err
}
return os.Rename(tmp, path)
return writePreserving(path, out)
}
func List(path string) ([]Caller, error) {
@@ -151,5 +149,49 @@ func Revoke(path, name string) error {
if err != nil {
return err
}
return os.WriteFile(path, raw, 0o640)
return writePreserving(path, raw)
}
func writePreserving(path string, data []byte) error {
mode := os.FileMode(0o640)
uid, gid := -1, -1
if st, err := os.Stat(path); err == nil {
mode = st.Mode()
if sys, ok := st.Sys().(*syscall.Stat_t); ok {
uid, gid = int(sys.Uid), int(sys.Gid)
}
}
f, err := os.CreateTemp(filepath.Dir(path), ".sift-keys-*.tmp")
if err != nil {
return err
}
tmp := f.Name()
ok := false
defer func() {
if !ok {
_ = os.Remove(tmp)
}
}()
if _, err := f.Write(data); err != nil {
_ = f.Close()
return err
}
if err := f.Chmod(mode.Perm()); err != nil {
_ = f.Close()
return err
}
if uid >= 0 {
if err := f.Chown(uid, gid); err != nil {
_ = f.Close()
return err
}
}
if err := f.Close(); err != nil {
return err
}
if err := os.Rename(tmp, path); err != nil {
return err
}
ok = true
return nil
}
internal/auth/auth_test.go
+19
View File
@@ -50,3 +50,22 @@ func TestAddLookupRevoke(t *testing.T) {
t.Fatal("revoked still valid")
}
}
func TestAddPreservesKeysMode(t *testing.T) {
dir := t.TempDir()
keys := filepath.Join(dir, "keys.toml")
if err := os.WriteFile(keys, []byte(""), 0o600); err != nil {
t.Fatal(err)
}
out := filepath.Join(dir, "k")
if err := Add(keys, "talos", "hermes-fleet", out); err != nil {
t.Fatal(err)
}
st, err := os.Stat(keys)
if err != nil {
t.Fatal(err)
}
if st.Mode().Perm() != 0o600 {
t.Fatalf("mode %o", st.Mode().Perm())
}
}
internal/config/config.go
+1
View File
@@ -21,6 +21,7 @@ type File struct {
type Dixie struct {
Chat string `toml:"chat"`
Embed string `toml:"embed"`
EmbedModel string `toml:"embed_model"`
MaxInflight int `toml:"max_inflight"`
QueueDepth int `toml:"queue_depth"`
QueueWait string `toml:"queue_wait"`
internal/digest/digest.go
+21 -10
View File
@@ -154,15 +154,16 @@ func (g *Gate) Release() {
}
type Engine struct {
ChatURL string
EmbedURL string
Model string
Thinking bool
Client *http.Client
Gate *Gate
Timeout time.Duration
Log *slog.Logger
Cache *dcache.DB
ChatURL string
EmbedURL string
EmbedModel string
Model string
Thinking bool
Client *http.Client
Gate *Gate
Timeout time.Duration
Log *slog.Logger
Cache *dcache.DB
mu sync.Mutex
Calls uint64
@@ -201,6 +202,13 @@ func (e *Engine) model() string {
return "ornith-1.5-9b-uncensored"
}
func (e *Engine) embedModel() string {
if e != nil && e.EmbedModel != "" {
return e.EmbedModel
}
return "honcho-embed"
}
func (e *Engine) noteFallback(reason string) {
if e == nil {
return
@@ -379,7 +387,7 @@ func (e *Engine) embed(ctx context.Context, inputs []string) ([][]float64, error
}
func (e *Engine) embedBatch(ctx context.Context, inputs []string) ([][]float64, error) {
payload, _ := json.Marshal(map[string]any{"input": inputs, "model": "qwen3-embedding-0.6b"})
payload, _ := json.Marshal(map[string]any{"input": inputs, "model": e.embedModel()})
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.EmbedURL, bytes.NewReader(payload))
if err != nil {
return nil, err
@@ -541,6 +549,9 @@ func countBudget(chunks []string, budget int) int {
return n
}
// CapBudget trims s to the digest input budget (SmallDoc estimated tokens).
func CapBudget(s string) string { return trimTokens(s, SmallDoc) }
func trimTokens(s string, max int) string {
r := []rune(s)
n := max * 4
internal/digest/digest_test.go
+39
View File
@@ -250,3 +250,42 @@ func TestChunkOrderAndBudget(t *testing.T) {
t.Fatal("document order")
}
}
func TestEmbedUsesConfiguredModel(t *testing.T) {
var got string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
Model string `json:"model"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
got = body.Model
_ = json.NewEncoder(w).Encode(map[string]any{
"data": []map[string]any{
{"index": 0, "embedding": []float64{1, 0}},
{"index": 1, "embedding": []float64{0, 1}},
},
})
}))
defer ts.Close()
e := &Engine{EmbedURL: ts.URL, EmbedModel: "honcho-embed", Client: ts.Client()}
if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
t.Fatal(err)
}
if got != "honcho-embed" {
t.Fatalf("model %q", got)
}
e.EmbedModel = "other-embed"
if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
t.Fatal(err)
}
if got != "other-embed" {
t.Fatalf("model %q", got)
}
e.EmbedModel = ""
if _, err := e.embedBatch(context.Background(), []string{"q", "chunk"}); err != nil {
t.Fatal(err)
}
if got != "honcho-embed" {
t.Fatalf("default model %q", got)
}
}
internal/recover/recover.go
+32 -6
View File
@@ -6,9 +6,11 @@ import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.wntrmute.dev/kyle/sift/internal/fetch"
)
@@ -61,9 +63,10 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
var env struct {
ArchivedSnapshots struct {
Closest struct {
Available bool `json:"available"`
URL string `json:"url"`
Timestamp string `json:"timestamp"`
Available bool `json:"available"`
URL string `json:"url"`
Timestamp string `json:"timestamp"`
Status json.RawMessage `json:"status"`
} `json:"closest"`
} `json:"archived_snapshots"`
}
@@ -74,11 +77,15 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
if !c.Available || c.URL == "" {
return nil, fmt.Errorf("wayback: unavailable")
}
if !waybackStatusOK(c.Status) {
return nil, fmt.Errorf("wayback: status %s", strings.TrimSpace(string(c.Status)))
}
u, err := url.Parse(c.URL)
if err != nil || u.Scheme != "https" || strings.ToLower(u.Hostname()) != "web.archive.org" {
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || strings.ToLower(u.Hostname()) != "web.archive.org" {
return nil, fmt.Errorf("wayback: bad snapshot url")
}
snapURL := insertID(c.URL)
u.Scheme = "https"
snapURL := insertID(u.String())
page, err := f.Fetch(ctx, snapURL)
if err != nil {
return nil, err
@@ -170,7 +177,26 @@ func mementoDatetime(res *fetch.Resource) string {
if res == nil || res.Header == nil {
return ""
}
return strings.TrimSpace(res.Header.Get("Memento-Datetime"))
raw := strings.TrimSpace(res.Header.Get("Memento-Datetime"))
if raw == "" {
return ""
}
t, err := http.ParseTime(raw)
if err != nil {
return ""
}
if t.After(time.Now().Add(24 * time.Hour)) {
return ""
}
return t.UTC().Format(time.RFC3339)
}
func waybackStatusOK(raw json.RawMessage) bool {
if len(bytesTrim(raw)) == 0 || string(raw) == "null" {
return true
}
s := strings.Trim(strings.TrimSpace(string(raw)), `"`)
return s == "200"
}
func hostOf(raw string) string {
internal/recover/recover_test.go
+58
View File
@@ -3,6 +3,7 @@ package recover
import (
"context"
"encoding/json"
"net/http"
"strings"
"testing"
"time"
@@ -166,3 +167,60 @@ func TestRedirectToPrivateBlocked(t *testing.T) {
t.Fatalf("err %v", err)
}
}
func TestWaybackHTTPRewrittenAnd204Skipped(t *testing.T) {
const fixture = `{"url":"https://www.ft.com/","archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://www.ft.com/","timestamp":"20240101120000","status":"204"}}}`
f := &fakeFetch{
body: map[string][]byte{
"https://archive.org/wayback/available": []byte(fixture),
"https://archive.ph/newest/": []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>"),
},
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
}
r, err := Try(context.Background(), f, "https://www.ft.com/")
if err != nil {
t.Fatal(err)
}
if r.Via != ViaToday {
t.Fatalf("via %s hits %v", r.Via, f.hits)
}
for _, h := range f.hits {
if strings.Contains(h, "web.archive.org") {
t.Fatalf("fetched 204 snapshot: %v", f.hits)
}
}
page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
avail := []byte(`{"archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://example.com/x","timestamp":"20240101120000","status":"200"}}}`)
f2 := &fakeFetch{
body: map[string][]byte{
"https://archive.org/wayback/available": avail,
"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
},
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
}
r, err = Try(context.Background(), f2, "https://example.com/x")
if err != nil {
t.Fatal(err)
}
if r.Via != ViaWayback {
t.Fatalf("via %s hits %v", r.Via, f2.hits)
}
}
func TestMementoDatetime(t *testing.T) {
h := make(http.Header)
h.Set("Memento-Datetime", "Wed, 01 Jan 2020 12:00:00 GMT")
got := mementoDatetime(&fetch.Resource{Header: h})
if got != "2020-01-01T12:00:00Z" {
t.Fatalf("got %q", got)
}
h.Set("Memento-Datetime", "Mon, 31 Dec 2029 00:00:00 GMT")
if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
t.Fatalf("future %q", got)
}
h.Set("Memento-Datetime", "not-a-date")
if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
t.Fatalf("garbage %q", got)
}
}