p2.1: honcho-embed, wayback http/204, memento, keys owner

Embed requests send dixie.embed_model (default honcho-embed).
Wayback accepts http://web.archive.org, rewrites to https, skips
non-200 closest.status. Memento-Datetime is parsed as an HTTP date
and future/garbage values become empty snapshot_at. keys add writes
via temp+fchown/chmod/rename so root does not clobber keys.toml.
Digest fallback is capped at the 6k-token budget.
This commit is contained in:
2026-09-23 02:41:46 -07:00
parent 2db001e0d7
commit 16a7d678d3
10 changed files with 248 additions and 30 deletions
+9 -1
View File
@@ -302,6 +302,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if s.Digest == nil {
warn = append(warn, "digest fell back to clean: unset")
mode = "clean"
content, warn = capDigestFallback(content, warn)
digMeta = map[string]any{"fallback": "unset"}
} else {
dr := s.Digest.Run(r.Context(), content, q, res.FinalURL)
@@ -309,7 +310,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if dr.Fallback != "" {
warn = append(warn, "digest fell back to clean: "+dr.Fallback)
mode = "clean"
content = clean.TruncateUTF8(content, s.OutCap)
content, warn = capDigestFallback(content, warn)
digMeta = map[string]any{"fallback": dr.Fallback, "model": dr.Model}
} else {
content = dr.Text
@@ -552,6 +553,13 @@ func (s *Server) release(caller, rawURL string) {
}
}
func capDigestFallback(content string, warn []string) (string, []string) {
if digest.EstTokens(content) <= digest.SmallDoc {
return content, warn
}
return digest.CapBudget(content), append(warn, "clean fallback truncated to digest budget")
}
func hostOf(raw string) string {
u, err := url.Parse(raw)
if err != nil {
+13
View File
@@ -16,6 +16,7 @@ import (
"time"
"git.wntrmute.dev/kyle/sift/internal/auth"
"git.wntrmute.dev/kyle/sift/internal/digest"
"git.wntrmute.dev/kyle/sift/internal/fetch"
"git.wntrmute.dev/kyle/sift/internal/guard"
)
@@ -156,6 +157,18 @@ func TestDigestFallbackWhenUnset(t *testing.T) {
}
}
func TestCapDigestFallback(t *testing.T) {
big := strings.Repeat("word ", 50000)
out, warn := capDigestFallback(big, nil)
if digest.EstTokens(out) > digest.SmallDoc {
t.Fatalf("tokens %d", digest.EstTokens(out))
}
joined := strings.Join(warn, " ")
if !strings.Contains(joined, "digest budget") {
t.Fatalf("warn %v", warn)
}
}
func TestNever502(t *testing.T) {
s, key := setupAPI(t, nil)
rec := postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})