p2: digest with in-sift FIFO, SQLite caches, Honcho context, recover

Dixie is capped (max_inflight 2, queue_depth 8, queue_wait 20s) and
never queued on llama-server. Streaming chat, thinking off, 45s
fallback to clean. Digest and chunk-embedding caches are
content-addressed SQLite (Honcho context is not cached). Recover
uses the guarded Fetcher. guardian-star is refused in code.

internal/recover/recover.go

@@ -0,0 +1,174 @@
+// Package recover is the server-side archive ladder. Every hop uses the
+// guarded Fetcher so SSRF still applies to archive redirects.
+package recover
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/fetch"
+)
+
+const bodyFloor = 400
+
+var (
+	interstitialTitle = regexp.MustCompile(`(?i)<title>[^<]*(Just a moment|Attention Required|Redirecting|Google Search)`)
+	metaRefresh       = regexp.MustCompile(`(?i)<meta[^>]+http-equiv=["']?refresh["'][^>]+url=["']?([^"'>\s]+)`)
+	jsLocation        = regexp.MustCompile(`(?i)(?:window\.)?location(?:\.href)?\s*=\s*["']([^"']+)["']`)
+)
+
+type Via string
+
+const (
+	ViaWayback Via = "wayback"
+	ViaToday   Via = "archive.today"
+)
+
+type Result struct {
+	Res         *fetch.Resource
+	Via         Via
+	SnapshotAt  string
+	OriginalURL string
+}
+
+type Fetcher interface {
+	Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error)
+}
+
+var archiveHosts = []string{"archive.ph", "archive.md", "archive.li", "archive.is"}
+
+// Try Wayback then archive.today. Fake successes advance to the next route.
+func Try(ctx context.Context, f Fetcher, original string) (*Result, error) {
+	if r, err := wayback(ctx, f, original); err == nil {
+		return r, nil
+	}
+	if r, err := archiveToday(ctx, f, original); err == nil {
+		return r, nil
+	}
+	return nil, fmt.Errorf("recover: no snapshot")
+}
+
+func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
+	avail := "https://archive.org/wayback/available?url=" + url.QueryEscape(original)
+	res, err := f.Fetch(ctx, avail)
+	if err != nil {
+		return nil, err
+	}
+	var env struct {
+		ArchivedSnapshots struct {
+			Closest struct {
+				Available bool   `json:"available"`
+				URL       string `json:"url"`
+				Timestamp string `json:"timestamp"`
+			} `json:"closest"`
+		} `json:"archived_snapshots"`
+	}
+	if err := json.Unmarshal(res.Body, &env); err != nil {
+		return nil, err
+	}
+	c := env.ArchivedSnapshots.Closest
+	if !c.Available || c.URL == "" {
+		return nil, fmt.Errorf("wayback: unavailable")
+	}
+	snapURL := insertID(c.URL)
+	page, err := f.Fetch(ctx, snapURL)
+	if err != nil {
+		return nil, err
+	}
+	if fakeSuccess(page, original) {
+		return nil, fmt.Errorf("wayback: fake success")
+	}
+	return &Result{Res: page, Via: ViaWayback, SnapshotAt: c.Timestamp, OriginalURL: original}, nil
+}
+
+func insertID(snap string) string {
+	// https://web.archive.org/web/20240101120000/https://example.com
+	// → …/web/20240101120000id_/https://example.com
+	const mark = "/web/"
+	i := strings.Index(snap, mark)
+	if i < 0 {
+		return snap
+	}
+	rest := snap[i+len(mark):]
+	j := strings.IndexByte(rest, '/')
+	if j < 0 {
+		return snap
+	}
+	ts := rest[:j]
+	if strings.Contains(ts, "id_") {
+		return snap
+	}
+	return snap[:i+len(mark)] + ts + "id_" + rest[j:]
+}
+
+func archiveToday(ctx context.Context, f Fetcher, original string) (*Result, error) {
+	var last error
+	for _, host := range archiveHosts {
+		u := "https://" + host + "/newest/" + original
+		res, err := f.Fetch(ctx, u)
+		if err != nil {
+			last = err
+			if fe, ok := err.(*fetch.Error); ok && (fe.Status == 429 || fe.Status >= 500) {
+				continue
+			}
+			continue
+		}
+		if fakeSuccess(res, original) {
+			last = fmt.Errorf("archive.today: fake success")
+			continue
+		}
+		return &Result{Res: res, Via: ViaToday, SnapshotAt: res.FetchedAt.UTC().Format(time.RFC3339), OriginalURL: original}, nil
+	}
+	if last == nil {
+		last = fmt.Errorf("archive.today: exhausted")
+	}
+	return nil, last
+}
+
+func fakeSuccess(res *fetch.Resource, original string) bool {
+	if res == nil {
+		return true
+	}
+	body := string(res.Body)
+	if len(bytesTrim(res.Body)) < bodyFloor {
+		return true
+	}
+	if interstitialTitle.MatchString(body) {
+		return true
+	}
+	if strings.Contains(strings.ToLower(body), "too many requests") && strings.Contains(strings.ToLower(res.FinalURL), "archive.") {
+		return true
+	}
+	origHost := hostOf(original)
+	if m := metaRefresh.FindStringSubmatch(body); len(m) > 1 {
+		if hostOf(m[1]) == origHost {
+			return true
+		}
+	}
+	if m := jsLocation.FindStringSubmatch(body); len(m) > 1 {
+		if hostOf(m[1]) == origHost {
+			return true
+		}
+	}
+	return false
+}
+
+func bytesTrim(b []byte) []byte {
+	return []byte(strings.TrimSpace(string(b)))
+}
+
+func hostOf(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return ""
+	}
+	return strings.ToLower(u.Hostname())
+}
+
+// InsertID is exported for tests.
+func InsertID(snap string) string { return insertID(snap) }