// Package recover is the server-side archive ladder. Every hop uses the // guarded Fetcher so SSRF still applies to archive redirects. package recover import ( "context" "encoding/json" "fmt" "net/url" "regexp" "strings" "git.wntrmute.dev/kyle/sift/internal/fetch" ) const bodyFloor = 400 var ( interstitialTitle = regexp.MustCompile(`(?i)[^<]*(Just a moment|Attention Required|Redirecting|Google Search)`) metaRefresh = regexp.MustCompile(`(?i)<meta[^>]+http-equiv=["']?refresh["'][^>]+url=["']?([^"'>\s]+)`) jsLocation = regexp.MustCompile(`(?i)(?:window\.)?location(?:\.href)?\s*=\s*["']([^"']+)["']`) ) type Via string const ( ViaWayback Via = "wayback" ViaToday Via = "archive.today" ) type Result struct { Res *fetch.Resource Via Via SnapshotAt string OriginalURL string } type Fetcher interface { Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error) } var archiveHosts = []string{"archive.ph", "archive.md", "archive.li", "archive.is"} // Try Wayback then archive.today. Fake successes advance to the next route. func Try(ctx context.Context, f Fetcher, original string) (*Result, error) { if r, err := wayback(ctx, f, original); err == nil { return r, nil } if r, err := archiveToday(ctx, f, original); err == nil { return r, nil } return nil, fmt.Errorf("recover: no snapshot") } func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) { avail := "https://archive.org/wayback/available?url=" + url.QueryEscape(original) res, err := f.Fetch(ctx, avail) if err != nil { return nil, err } var env struct { ArchivedSnapshots struct { Closest struct { Available bool `json:"available"` URL string `json:"url"` Timestamp string `json:"timestamp"` } `json:"closest"` } `json:"archived_snapshots"` } if err := json.Unmarshal(res.Body, &env); err != nil { return nil, err } c := env.ArchivedSnapshots.Closest if !c.Available || c.URL == "" { return nil, fmt.Errorf("wayback: unavailable") } u, err := url.Parse(c.URL) if err != nil || u.Scheme != "https" || strings.ToLower(u.Hostname()) != "web.archive.org" { return nil, fmt.Errorf("wayback: bad snapshot url") } snapURL := insertID(c.URL) page, err := f.Fetch(ctx, snapURL) if err != nil { return nil, err } if fakeSuccess(page, original) { return nil, fmt.Errorf("wayback: fake success") } return &Result{Res: page, Via: ViaWayback, SnapshotAt: c.Timestamp, OriginalURL: original}, nil } func insertID(snap string) string { // https://web.archive.org/web/20240101120000/https://example.com // → …/web/20240101120000id_/https://example.com const mark = "/web/" i := strings.Index(snap, mark) if i < 0 { return snap } rest := snap[i+len(mark):] j := strings.IndexByte(rest, '/') if j < 0 { return snap } ts := rest[:j] if strings.Contains(ts, "id_") { return snap } return snap[:i+len(mark)] + ts + "id_" + rest[j:] } func archiveToday(ctx context.Context, f Fetcher, original string) (*Result, error) { var last error for _, host := range archiveHosts { u := "https://" + host + "/newest/" + original res, err := f.Fetch(ctx, u) if err != nil { last = err if fe, ok := err.(*fetch.Error); ok && (fe.Status == 429 || fe.Status >= 500) { continue } continue } if fakeSuccess(res, original) { last = fmt.Errorf("archive.today: fake success") continue } at := mementoDatetime(res) return &Result{Res: res, Via: ViaToday, SnapshotAt: at, OriginalURL: original}, nil } if last == nil { last = fmt.Errorf("archive.today: exhausted") } return nil, last } func fakeSuccess(res *fetch.Resource, original string) bool { if res == nil { return true } body := string(res.Body) if len(bytesTrim(res.Body)) < bodyFloor { return true } if interstitialTitle.MatchString(body) { return true } if strings.Contains(strings.ToLower(body), "too many requests") && strings.Contains(strings.ToLower(res.FinalURL), "archive.") { return true } origHost := hostOf(original) if m := metaRefresh.FindStringSubmatch(body); len(m) > 1 { if hostOf(m[1]) == origHost { return true } } if m := jsLocation.FindStringSubmatch(body); len(m) > 1 { if hostOf(m[1]) == origHost { return true } } return false } func bytesTrim(b []byte) []byte { return []byte(strings.TrimSpace(string(b))) } func mementoDatetime(res *fetch.Resource) string { if res == nil || res.Header == nil { return "" } return strings.TrimSpace(res.Header.Get("Memento-Datetime")) } func hostOf(raw string) string { u, err := url.Parse(raw) if err != nil { return "" } return strings.ToLower(u.Hostname()) } // InsertID is exported for tests. func InsertID(snap string) string { return insertID(snap) }