p2: digest with in-sift FIFO, SQLite caches, Honcho context, recover

Dixie is capped (max_inflight 2, queue_depth 8, queue_wait 20s) and never queued on llama-server. Streaming chat, thinking off, 45s fallback to clean. Digest and chunk-embedding caches are content-addressed SQLite (Honcho context is not cached). Recover uses the guarded Fetcher. guardian-star is refused in code.
2026-09-22 23:54:49 -07:00
parent 505e113b64
commit 2f9d3d97e6
2023 changed files with 3815943 additions and 30 deletions
@@ -0,0 +1,152 @@
+package recover
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/fetch"
+)
+
+type fakeFetch struct {
+	body map[string][]byte
+	ct   map[string]string
+	err  map[string]error
+	hits []string
+}
+
+func (f *fakeFetch) Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error) {
+	f.hits = append(f.hits, rawURL)
+	if e, ok := f.err[rawURL]; ok {
+		return nil, e
+	}
+	for k, b := range f.body {
+		if strings.HasPrefix(rawURL, k) || rawURL == k {
+			ct := f.ct[k]
+			if ct == "" {
+				ct = "text/html"
+			}
+			return &fetch.Resource{
+				URL: rawURL, FinalURL: rawURL, Status: 200,
+				ContentType: ct, Body: b, FetchedAt: time.Now(),
+			}, nil
+		}
+	}
+	return nil, &fetch.Error{Status: 404, Msg: "missing " + rawURL}
+}
+
+func TestInsertID(t *testing.T) {
+	in := "https://web.archive.org/web/20240101120000/https://example.com/x"
+	got := InsertID(in)
+	if !strings.Contains(got, "20240101120000id_/") {
+		t.Fatalf("%s", got)
+	}
+}
+
+func TestWaybackAvailable(t *testing.T) {
+	snap := "https://web.archive.org/web/20240101120000/https://example.com/x"
+	avail, _ := json.Marshal(map[string]any{
+		"archived_snapshots": map[string]any{
+			"closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"},
+		},
+	})
+	page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
+	f := &fakeFetch{
+		body: map[string][]byte{
+			"https://archive.org/wayback/available":                               avail,
+			"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
+		},
+		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
+	}
+	r, err := Try(context.Background(), f, "https://example.com/x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.Via != ViaWayback || r.SnapshotAt != "20240101120000" {
+		t.Fatalf("%+v", r)
+	}
+}
+
+func TestWaybackUnavailableGoesToArchiveToday(t *testing.T) {
+	avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
+	page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
+	f := &fakeFetch{
+		body: map[string][]byte{
+			"https://archive.org/wayback/available": avail,
+			"https://archive.ph/newest/":            page,
+		},
+		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
+	}
+	r, err := Try(context.Background(), f, "https://example.com/x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.Via != ViaToday {
+		t.Fatalf("via %s", r.Via)
+	}
+}
+
+func TestFakeSuccessMetaRefresh(t *testing.T) {
+	body := []byte(`<html><head><meta http-equiv="refresh" content="0;url=https://example.com/x"><title>Redirecting</title></head><body>` + strings.Repeat("x", 500) + `</body></html>`)
+	if !fakeSuccess(&fetch.Resource{Body: body, FinalURL: "https://web.archive.org/foo"}, "https://example.com/x") {
+		t.Fatal("want fake")
+	}
+}
+
+func TestFakeSuccessJustAMoment(t *testing.T) {
+	body := []byte(`<html><head><title>Just a moment...</title></head><body>` + strings.Repeat("x", 500) + `</body></html>`)
+	if !fakeSuccess(&fetch.Resource{Body: body}, "https://example.com") {
+		t.Fatal("want fake")
+	}
+}
+
+func TestFakeSuccessTinyBody(t *testing.T) {
+	if !fakeSuccess(&fetch.Resource{Body: []byte("short")}, "https://example.com") {
+		t.Fatal("want fake")
+	}
+}
+
+func TestArchiveTodayRotateOn429(t *testing.T) {
+	page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
+	avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
+	f := &fakeFetch{
+		body: map[string][]byte{
+			"https://archive.org/wayback/available": avail,
+			"https://archive.md/newest/":            page,
+		},
+		ct:  map[string]string{"https://archive.org/wayback/available": "application/json"},
+		err: map[string]error{"https://archive.ph/newest/https://example.com/x": &fetch.Error{Status: 429, Msg: "busy"}},
+	}
+	r, err := Try(context.Background(), f, "https://example.com/x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.Via != ViaToday {
+		t.Fatalf("via %s hits %v", r.Via, f.hits)
+	}
+}
+
+func TestRedirectToPrivateBlocked(t *testing.T) {
+	snap := "https://web.archive.org/web/20240101120000/https://example.com/x"
+	avail, _ := json.Marshal(map[string]any{
+		"archived_snapshots": map[string]any{
+			"closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"},
+		},
+	})
+	f := &fakeFetch{
+		body: map[string][]byte{"https://archive.org/wayback/available": avail},
+		ct:   map[string]string{"https://archive.org/wayback/available": "application/json"},
+		err: map[string]error{
+			"https://web.archive.org/web/20240101120000id_/https://example.com/x": &fetch.Error{Status: 403, Msg: "blocked: private address 10.0.0.5"},
+		},
+	}
+	_, err := Try(context.Background(), f, "https://example.com/x")
+	if err == nil {
+		t.Fatal("want error")
+	}
+	if !strings.Contains(err.Error(), "blocked:") && !strings.Contains(err.Error(), "no snapshot") && !strings.Contains(err.Error(), "private") {
+		t.Fatalf("err %v", err)
+	}
+}