// Package recover is the server-side archive ladder. Every hop uses the
// guarded Fetcher so SSRF still applies to archive redirects.
package recover
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.wntrmute.dev/kyle/sift/internal/fetch"
)
const bodyFloor = 400
var (
interstitialTitle = regexp.MustCompile(`(?i)
[^<]*(Just a moment|Attention Required|Redirecting|Google Search)`)
metaRefresh = regexp.MustCompile(`(?i)]+http-equiv=["']?refresh["'][^>]+url=["']?([^"'>\s]+)`)
jsLocation = regexp.MustCompile(`(?i)(?:window\.)?location(?:\.href)?\s*=\s*["']([^"']+)["']`)
)
type Via string
const (
ViaWayback Via = "wayback"
ViaToday Via = "archive.today"
)
type Result struct {
Res *fetch.Resource
Via Via
SnapshotAt string
OriginalURL string
}
type Fetcher interface {
Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error)
}
var archiveHosts = []string{"archive.ph", "archive.md", "archive.li", "archive.is"}
// Try Wayback then archive.today. Fake successes advance to the next route.
func Try(ctx context.Context, f Fetcher, original string) (*Result, error) {
if r, err := wayback(ctx, f, original); err == nil {
return r, nil
}
if r, err := archiveToday(ctx, f, original); err == nil {
return r, nil
}
return nil, fmt.Errorf("recover: no snapshot")
}
func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
avail := "https://archive.org/wayback/available?url=" + url.QueryEscape(original)
res, err := f.Fetch(ctx, avail)
if err != nil {
return nil, err
}
var env struct {
ArchivedSnapshots struct {
Closest struct {
Available bool `json:"available"`
URL string `json:"url"`
Timestamp string `json:"timestamp"`
Status json.RawMessage `json:"status"`
} `json:"closest"`
} `json:"archived_snapshots"`
}
if err := json.Unmarshal(res.Body, &env); err != nil {
return nil, err
}
c := env.ArchivedSnapshots.Closest
if !c.Available || c.URL == "" {
return nil, fmt.Errorf("wayback: unavailable")
}
if !waybackStatusOK(c.Status) {
return nil, fmt.Errorf("wayback: status %s", strings.TrimSpace(string(c.Status)))
}
u, err := url.Parse(c.URL)
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || strings.ToLower(u.Hostname()) != "web.archive.org" {
return nil, fmt.Errorf("wayback: bad snapshot url")
}
u.Scheme = "https"
snapURL := insertID(u.String())
page, err := f.Fetch(ctx, snapURL)
if err != nil {
return nil, err
}
if fakeSuccess(page, original) {
return nil, fmt.Errorf("wayback: fake success")
}
return &Result{Res: page, Via: ViaWayback, SnapshotAt: c.Timestamp, OriginalURL: original}, nil
}
func insertID(snap string) string {
// https://web.archive.org/web/20240101120000/https://example.com
// → …/web/20240101120000id_/https://example.com
const mark = "/web/"
i := strings.Index(snap, mark)
if i < 0 {
return snap
}
rest := snap[i+len(mark):]
j := strings.IndexByte(rest, '/')
if j < 0 {
return snap
}
ts := rest[:j]
if strings.Contains(ts, "id_") {
return snap
}
return snap[:i+len(mark)] + ts + "id_" + rest[j:]
}
func archiveToday(ctx context.Context, f Fetcher, original string) (*Result, error) {
var last error
for _, host := range archiveHosts {
u := "https://" + host + "/newest/" + original
res, err := f.Fetch(ctx, u)
if err != nil {
last = err
if fe, ok := err.(*fetch.Error); ok && (fe.Status == 429 || fe.Status >= 500) {
continue
}
continue
}
if fakeSuccess(res, original) {
last = fmt.Errorf("archive.today: fake success")
continue
}
at := mementoDatetime(res)
return &Result{Res: res, Via: ViaToday, SnapshotAt: at, OriginalURL: original}, nil
}
if last == nil {
last = fmt.Errorf("archive.today: exhausted")
}
return nil, last
}
func fakeSuccess(res *fetch.Resource, original string) bool {
if res == nil {
return true
}
body := string(res.Body)
if len(bytesTrim(res.Body)) < bodyFloor {
return true
}
if interstitialTitle.MatchString(body) {
return true
}
if strings.Contains(strings.ToLower(body), "too many requests") && strings.Contains(strings.ToLower(res.FinalURL), "archive.") {
return true
}
origHost := hostOf(original)
if m := metaRefresh.FindStringSubmatch(body); len(m) > 1 {
if hostOf(m[1]) == origHost {
return true
}
}
if m := jsLocation.FindStringSubmatch(body); len(m) > 1 {
if hostOf(m[1]) == origHost {
return true
}
}
return false
}
func bytesTrim(b []byte) []byte {
return []byte(strings.TrimSpace(string(b)))
}
func mementoDatetime(res *fetch.Resource) string {
if res == nil || res.Header == nil {
return ""
}
raw := strings.TrimSpace(res.Header.Get("Memento-Datetime"))
if raw == "" {
return ""
}
t, err := http.ParseTime(raw)
if err != nil {
return ""
}
if t.After(time.Now().Add(24 * time.Hour)) {
return ""
}
return t.UTC().Format(time.RFC3339)
}
func waybackStatusOK(raw json.RawMessage) bool {
if len(bytesTrim(raw)) == 0 || string(raw) == "null" {
return true
}
s := strings.Trim(strings.TrimSpace(string(raw)), `"`)
return s == "200"
}
func hostOf(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return ""
}
return strings.ToLower(u.Hostname())
}
// InsertID is exported for tests.
func InsertID(snap string) string { return insertID(snap) }