kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

p2: digest with in-sift FIFO, SQLite caches, Honcho context, recover

Browse Source 
Dixie is capped (max_inflight 2, queue_depth 8, queue_wait 20s) and
never queued on llama-server. Streaming chat, thinking off, 45s
fallback to clean. Digest and chunk-embedding caches are
content-addressed SQLite (Honcho context is not cached). Recover
uses the guarded Fetcher. guardian-star is refused in code.
This commit is contained in:
kyle
2026-09-22 23:54:49 -07:00
parent 505e113b64
commit 2f9d3d97e6
2023 changed files with 3815943 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/recover/recover.go
+174
View File
@@ -0,0 +1,174 @@
// Package recover is the server-side archive ladder. Every hop uses the
// guarded Fetcher so SSRF still applies to archive redirects.
package recover
import (
"context"
"encoding/json"
"fmt"
"net/url"
"regexp"
"strings"
"time"
"git.wntrmute.dev/kyle/sift/internal/fetch"
)
const bodyFloor = 400
var (
interstitialTitle = regexp.MustCompile(`(?i)<title>[^<]*(Just a moment|Attention Required|Redirecting|Google Search)`)
metaRefresh = regexp.MustCompile(`(?i)<meta[^>]+http-equiv=["']?refresh["'][^>]+url=["']?([^"'>\s]+)`)
jsLocation = regexp.MustCompile(`(?i)(?:window\.)?location(?:\.href)?\s*=\s*["']([^"']+)["']`)
)
type Via string
const (
ViaWayback Via = "wayback"
ViaToday Via = "archive.today"
)
type Result struct {
Res *fetch.Resource
Via Via
SnapshotAt string
OriginalURL string
}
type Fetcher interface {
Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error)
}
var archiveHosts = []string{"archive.ph", "archive.md", "archive.li", "archive.is"}
// Try Wayback then archive.today. Fake successes advance to the next route.
func Try(ctx context.Context, f Fetcher, original string) (*Result, error) {
if r, err := wayback(ctx, f, original); err == nil {
return r, nil
}
if r, err := archiveToday(ctx, f, original); err == nil {
return r, nil
}
return nil, fmt.Errorf("recover: no snapshot")
}
func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
avail := "https://archive.org/wayback/available?url=" + url.QueryEscape(original)
res, err := f.Fetch(ctx, avail)
if err != nil {
return nil, err
}
var env struct {
ArchivedSnapshots struct {
Closest struct {
Available bool `json:"available"`
URL string `json:"url"`
Timestamp string `json:"timestamp"`
} `json:"closest"`
} `json:"archived_snapshots"`
}
if err := json.Unmarshal(res.Body, &env); err != nil {
return nil, err
}
c := env.ArchivedSnapshots.Closest
if !c.Available || c.URL == "" {
return nil, fmt.Errorf("wayback: unavailable")
}
snapURL := insertID(c.URL)
page, err := f.Fetch(ctx, snapURL)
if err != nil {
return nil, err
}
if fakeSuccess(page, original) {
return nil, fmt.Errorf("wayback: fake success")
}
return &Result{Res: page, Via: ViaWayback, SnapshotAt: c.Timestamp, OriginalURL: original}, nil
}
func insertID(snap string) string {
// https://web.archive.org/web/20240101120000/https://example.com
// → …/web/20240101120000id_/https://example.com
const mark = "/web/"
i := strings.Index(snap, mark)
if i < 0 {
return snap
}
rest := snap[i+len(mark):]
j := strings.IndexByte(rest, '/')
if j < 0 {
return snap
}
ts := rest[:j]
if strings.Contains(ts, "id_") {
return snap
}
return snap[:i+len(mark)] + ts + "id_" + rest[j:]
}
func archiveToday(ctx context.Context, f Fetcher, original string) (*Result, error) {
var last error
for _, host := range archiveHosts {
u := "https://" + host + "/newest/" + original
res, err := f.Fetch(ctx, u)
if err != nil {
last = err
if fe, ok := err.(*fetch.Error); ok && (fe.Status == 429 || fe.Status >= 500) {
continue
}
continue
}
if fakeSuccess(res, original) {
last = fmt.Errorf("archive.today: fake success")
continue
}
return &Result{Res: res, Via: ViaToday, SnapshotAt: res.FetchedAt.UTC().Format(time.RFC3339), OriginalURL: original}, nil
}
if last == nil {
last = fmt.Errorf("archive.today: exhausted")
}
return nil, last
}
func fakeSuccess(res *fetch.Resource, original string) bool {
if res == nil {
return true
}
body := string(res.Body)
if len(bytesTrim(res.Body)) < bodyFloor {
return true
}
if interstitialTitle.MatchString(body) {
return true
}
if strings.Contains(strings.ToLower(body), "too many requests") && strings.Contains(strings.ToLower(res.FinalURL), "archive.") {
return true
}
origHost := hostOf(original)
if m := metaRefresh.FindStringSubmatch(body); len(m) > 1 {
if hostOf(m[1]) == origHost {
return true
}
}
if m := jsLocation.FindStringSubmatch(body); len(m) > 1 {
if hostOf(m[1]) == origHost {
return true
}
}
return false
}
func bytesTrim(b []byte) []byte {
return []byte(strings.TrimSpace(string(b)))
}
func hostOf(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return ""
}
return strings.ToLower(u.Hostname())
}
// InsertID is exported for tests.
func InsertID(snap string) string { return insertID(snap) }
internal/recover/recover_test.go
+152
View File
@@ -0,0 +1,152 @@
package recover
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.wntrmute.dev/kyle/sift/internal/fetch"
)
type fakeFetch struct {
body map[string][]byte
ct map[string]string
err map[string]error
hits []string
}
func (f *fakeFetch) Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error) {
f.hits = append(f.hits, rawURL)
if e, ok := f.err[rawURL]; ok {
return nil, e
}
for k, b := range f.body {
if strings.HasPrefix(rawURL, k) || rawURL == k {
ct := f.ct[k]
if ct == "" {
ct = "text/html"
}
return &fetch.Resource{
URL: rawURL, FinalURL: rawURL, Status: 200,
ContentType: ct, Body: b, FetchedAt: time.Now(),
}, nil
}
}
return nil, &fetch.Error{Status: 404, Msg: "missing " + rawURL}
}
func TestInsertID(t *testing.T) {
in := "https://web.archive.org/web/20240101120000/https://example.com/x"
got := InsertID(in)
if !strings.Contains(got, "20240101120000id_/") {
t.Fatalf("%s", got)
}
}
func TestWaybackAvailable(t *testing.T) {
snap := "https://web.archive.org/web/20240101120000/https://example.com/x"
avail, _ := json.Marshal(map[string]any{
"archived_snapshots": map[string]any{
"closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"},
},
})
page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
f := &fakeFetch{
body: map[string][]byte{
"https://archive.org/wayback/available": avail,
"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
},
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
}
r, err := Try(context.Background(), f, "https://example.com/x")
if err != nil {
t.Fatal(err)
}
if r.Via != ViaWayback || r.SnapshotAt != "20240101120000" {
t.Fatalf("%+v", r)
}
}
func TestWaybackUnavailableGoesToArchiveToday(t *testing.T) {
avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
f := &fakeFetch{
body: map[string][]byte{
"https://archive.org/wayback/available": avail,
"https://archive.ph/newest/": page,
},
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
}
r, err := Try(context.Background(), f, "https://example.com/x")
if err != nil {
t.Fatal(err)
}
if r.Via != ViaToday {
t.Fatalf("via %s", r.Via)
}
}
func TestFakeSuccessMetaRefresh(t *testing.T) {
body := []byte(`<html><head><meta http-equiv="refresh" content="0;url=https://example.com/x"><title>Redirecting</title></head><body>` + strings.Repeat("x", 500) + `</body></html>`)
if !fakeSuccess(&fetch.Resource{Body: body, FinalURL: "https://web.archive.org/foo"}, "https://example.com/x") {
t.Fatal("want fake")
}
}
func TestFakeSuccessJustAMoment(t *testing.T) {
body := []byte(`<html><head><title>Just a moment...</title></head><body>` + strings.Repeat("x", 500) + `</body></html>`)
if !fakeSuccess(&fetch.Resource{Body: body}, "https://example.com") {
t.Fatal("want fake")
}
}
func TestFakeSuccessTinyBody(t *testing.T) {
if !fakeSuccess(&fetch.Resource{Body: []byte("short")}, "https://example.com") {
t.Fatal("want fake")
}
}
func TestArchiveTodayRotateOn429(t *testing.T) {
page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
f := &fakeFetch{
body: map[string][]byte{
"https://archive.org/wayback/available": avail,
"https://archive.md/newest/": page,
},
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
err: map[string]error{"https://archive.ph/newest/https://example.com/x": &fetch.Error{Status: 429, Msg: "busy"}},
}
r, err := Try(context.Background(), f, "https://example.com/x")
if err != nil {
t.Fatal(err)
}
if r.Via != ViaToday {
t.Fatalf("via %s hits %v", r.Via, f.hits)
}
}
func TestRedirectToPrivateBlocked(t *testing.T) {
snap := "https://web.archive.org/web/20240101120000/https://example.com/x"
avail, _ := json.Marshal(map[string]any{
"archived_snapshots": map[string]any{
"closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"},
},
})
f := &fakeFetch{
body: map[string][]byte{"https://archive.org/wayback/available": avail},
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
err: map[string]error{
"https://web.archive.org/web/20240101120000id_/https://example.com/x": &fetch.Error{Status: 403, Msg: "blocked: private address 10.0.0.5"},
},
}
_, err := Try(context.Background(), f, "https://example.com/x")
if err == nil {
t.Fatal("want error")
}
if !strings.Contains(err.Error(), "blocked:") && !strings.Contains(err.Error(), "no snapshot") && !strings.Contains(err.Error(), "private") {
t.Fatalf("err %v", err)
}
}