sift/internal/recover/recover.go

// Package recover is the server-side archive ladder. Every hop uses the
// guarded Fetcher so SSRF still applies to archive redirects.
package recover

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"git.wntrmute.dev/kyle/sift/internal/fetch"
)

const bodyFloor = 400

var (
	interstitialTitle = regexp.MustCompile(`(?i)<title>[^<]*(Just a moment|Attention Required|Redirecting|Google Search)`)
	metaRefresh       = regexp.MustCompile(`(?i)<meta[^>]+http-equiv=["']?refresh["'][^>]+url=["']?([^"'>\s]+)`)
	jsLocation        = regexp.MustCompile(`(?i)(?:window\.)?location(?:\.href)?\s*=\s*["']([^"']+)["']`)
)

type Via string

const (
	ViaWayback Via = "wayback"
	ViaToday   Via = "archive.today"
)

type Result struct {
	Res         *fetch.Resource
	Via         Via
	SnapshotAt  string
	OriginalURL string
}

type Fetcher interface {
	Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error)
}

var archiveHosts = []string{"archive.ph", "archive.md", "archive.li", "archive.is"}

// Try Wayback then archive.today. Fake successes advance to the next route.
func Try(ctx context.Context, f Fetcher, original string) (*Result, error) {
	if r, err := wayback(ctx, f, original); err == nil {
		return r, nil
	}
	if r, err := archiveToday(ctx, f, original); err == nil {
		return r, nil
	}
	return nil, fmt.Errorf("recover: no snapshot")
}

func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
	avail := "https://archive.org/wayback/available?url=" + url.QueryEscape(original)
	res, err := f.Fetch(ctx, avail)
	if err != nil {
		return nil, err
	}
	var env struct {
		ArchivedSnapshots struct {
			Closest struct {
				Available bool            `json:"available"`
				URL       string          `json:"url"`
				Timestamp string          `json:"timestamp"`
				Status    json.RawMessage `json:"status"`
			} `json:"closest"`
		} `json:"archived_snapshots"`
	}
	if err := json.Unmarshal(res.Body, &env); err != nil {
		return nil, err
	}
	c := env.ArchivedSnapshots.Closest
	if !c.Available || c.URL == "" {
		return nil, fmt.Errorf("wayback: unavailable")
	}
	if !waybackStatusOK(c.Status) {
		return nil, fmt.Errorf("wayback: status %s", strings.TrimSpace(string(c.Status)))
	}
	u, err := url.Parse(c.URL)
	if err != nil || (u.Scheme != "https" && u.Scheme != "http") || strings.ToLower(u.Hostname()) != "web.archive.org" {
		return nil, fmt.Errorf("wayback: bad snapshot url")
	}
	u.Scheme = "https"
	snapURL := insertID(u.String())
	page, err := f.Fetch(ctx, snapURL)
	if err != nil {
		return nil, err
	}
	if fakeSuccess(page, original) {
		return nil, fmt.Errorf("wayback: fake success")
	}
	return &Result{Res: page, Via: ViaWayback, SnapshotAt: c.Timestamp, OriginalURL: original}, nil
}

func insertID(snap string) string {
	// https://web.archive.org/web/20240101120000/https://example.com
	// → …/web/20240101120000id_/https://example.com
	const mark = "/web/"
	i := strings.Index(snap, mark)
	if i < 0 {
		return snap
	}
	rest := snap[i+len(mark):]
	j := strings.IndexByte(rest, '/')
	if j < 0 {
		return snap
	}
	ts := rest[:j]
	if strings.Contains(ts, "id_") {
		return snap
	}
	return snap[:i+len(mark)] + ts + "id_" + rest[j:]
}

func archiveToday(ctx context.Context, f Fetcher, original string) (*Result, error) {
	var last error
	for _, host := range archiveHosts {
		u := "https://" + host + "/newest/" + original
		res, err := f.Fetch(ctx, u)
		if err != nil {
			last = err
			if fe, ok := err.(*fetch.Error); ok && (fe.Status == 429 || fe.Status >= 500) {
				continue
			}
			continue
		}
		if fakeSuccess(res, original) {
			last = fmt.Errorf("archive.today: fake success")
			continue
		}
		at := mementoDatetime(res)
		return &Result{Res: res, Via: ViaToday, SnapshotAt: at, OriginalURL: original}, nil
	}
	if last == nil {
		last = fmt.Errorf("archive.today: exhausted")
	}
	return nil, last
}

func fakeSuccess(res *fetch.Resource, original string) bool {
	if res == nil {
		return true
	}
	body := string(res.Body)
	if len(bytesTrim(res.Body)) < bodyFloor {
		return true
	}
	if interstitialTitle.MatchString(body) {
		return true
	}
	if strings.Contains(strings.ToLower(body), "too many requests") && strings.Contains(strings.ToLower(res.FinalURL), "archive.") {
		return true
	}
	origHost := hostOf(original)
	if m := metaRefresh.FindStringSubmatch(body); len(m) > 1 {
		if hostOf(m[1]) == origHost {
			return true
		}
	}
	if m := jsLocation.FindStringSubmatch(body); len(m) > 1 {
		if hostOf(m[1]) == origHost {
			return true
		}
	}
	return false
}

func bytesTrim(b []byte) []byte {
	return []byte(strings.TrimSpace(string(b)))
}

func mementoDatetime(res *fetch.Resource) string {
	if res == nil || res.Header == nil {
		return ""
	}
	raw := strings.TrimSpace(res.Header.Get("Memento-Datetime"))
	if raw == "" {
		return ""
	}
	t, err := http.ParseTime(raw)
	if err != nil {
		return ""
	}
	if t.After(time.Now().Add(24 * time.Hour)) {
		return ""
	}
	return t.UTC().Format(time.RFC3339)
}

func waybackStatusOK(raw json.RawMessage) bool {
	if len(bytesTrim(raw)) == 0 || string(raw) == "null" {
		return true
	}
	s := strings.Trim(strings.TrimSpace(string(raw)), `"`)
	return s == "200"
}

func hostOf(raw string) string {
	u, err := url.Parse(raw)
	if err != nil {
		return ""
	}
	return strings.ToLower(u.Hostname())
}

// InsertID is exported for tests.
func InsertID(snap string) string { return insertID(snap) }