sift/internal/recover/recover_test.go

package recover

import (
	"context"
	"encoding/json"
	"net/http"
	"strings"
	"testing"
	"time"

	"git.wntrmute.dev/kyle/sift/internal/fetch"
)

type fakeFetch struct {
	body map[string][]byte
	ct   map[string]string
	err  map[string]error
	hits []string
}

func (f *fakeFetch) Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error) {
	f.hits = append(f.hits, rawURL)
	if e, ok := f.err[rawURL]; ok {
		return nil, e
	}
	for k, b := range f.body {
		if strings.HasPrefix(rawURL, k) || rawURL == k {
			ct := f.ct[k]
			if ct == "" {
				ct = "text/html"
			}
			return &fetch.Resource{
				URL: rawURL, FinalURL: rawURL, Status: 200,
				ContentType: ct, Body: b, FetchedAt: time.Now(),
			}, nil
		}
	}
	return nil, &fetch.Error{Status: 404, Msg: "missing " + rawURL}
}

func TestInsertID(t *testing.T) {
	in := "https://web.archive.org/web/20240101120000/https://example.com/x"
	got := InsertID(in)
	if !strings.Contains(got, "20240101120000id_/") {
		t.Fatalf("%s", got)
	}
}

func TestWaybackAvailable(t *testing.T) {
	snap := "https://web.archive.org/web/20240101120000/https://example.com/x"
	avail, _ := json.Marshal(map[string]any{
		"archived_snapshots": map[string]any{
			"closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"},
		},
	})
	page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
	f := &fakeFetch{
		body: map[string][]byte{
			"https://archive.org/wayback/available":                               avail,
			"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
		},
		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
	}
	r, err := Try(context.Background(), f, "https://example.com/x")
	if err != nil {
		t.Fatal(err)
	}
	if r.Via != ViaWayback || r.SnapshotAt != "20240101120000" {
		t.Fatalf("%+v", r)
	}
}

func TestWaybackRejectsForeignHost(t *testing.T) {
	avail, _ := json.Marshal(map[string]any{
		"archived_snapshots": map[string]any{
			"closest": map[string]any{"available": true, "url": "https://evil.example/x", "timestamp": "20240101120000"},
		},
	})
	f := &fakeFetch{
		body: map[string][]byte{"https://archive.org/wayback/available": avail},
		ct:   map[string]string{"https://archive.org/wayback/available": "application/json"},
	}
	_, err := Try(context.Background(), f, "https://example.com/x")
	if err == nil {
		t.Fatal("want reject")
	}
}

func TestWaybackUnavailableGoesToArchiveToday(t *testing.T) {
	avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
	page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
	f := &fakeFetch{
		body: map[string][]byte{
			"https://archive.org/wayback/available": avail,
			"https://archive.ph/newest/":            page,
		},
		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
	}
	r, err := Try(context.Background(), f, "https://example.com/x")
	if err != nil {
		t.Fatal(err)
	}
	if r.Via != ViaToday {
		t.Fatalf("via %s", r.Via)
	}
}

func TestFakeSuccessMetaRefresh(t *testing.T) {
	body := []byte(`<html><head><meta http-equiv="refresh" content="0;url=https://example.com/x"><title>Redirecting</title></head><body>` + strings.Repeat("x", 500) + `</body></html>`)
	if !fakeSuccess(&fetch.Resource{Body: body, FinalURL: "https://web.archive.org/foo"}, "https://example.com/x") {
		t.Fatal("want fake")
	}
}

func TestFakeSuccessJustAMoment(t *testing.T) {
	body := []byte(`<html><head><title>Just a moment...</title></head><body>` + strings.Repeat("x", 500) + `</body></html>`)
	if !fakeSuccess(&fetch.Resource{Body: body}, "https://example.com") {
		t.Fatal("want fake")
	}
}

func TestFakeSuccessTinyBody(t *testing.T) {
	if !fakeSuccess(&fetch.Resource{Body: []byte("short")}, "https://example.com") {
		t.Fatal("want fake")
	}
}

func TestArchiveTodayRotateOn429(t *testing.T) {
	page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
	avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
	f := &fakeFetch{
		body: map[string][]byte{
			"https://archive.org/wayback/available": avail,
			"https://archive.md/newest/":            page,
		},
		ct:  map[string]string{"https://archive.org/wayback/available": "application/json"},
		err: map[string]error{"https://archive.ph/newest/https://example.com/x": &fetch.Error{Status: 429, Msg: "busy"}},
	}
	r, err := Try(context.Background(), f, "https://example.com/x")
	if err != nil {
		t.Fatal(err)
	}
	if r.Via != ViaToday {
		t.Fatalf("via %s hits %v", r.Via, f.hits)
	}
}

func TestRedirectToPrivateBlocked(t *testing.T) {
	snap := "https://web.archive.org/web/20240101120000/https://example.com/x"
	avail, _ := json.Marshal(map[string]any{
		"archived_snapshots": map[string]any{
			"closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"},
		},
	})
	f := &fakeFetch{
		body: map[string][]byte{"https://archive.org/wayback/available": avail},
		ct:   map[string]string{"https://archive.org/wayback/available": "application/json"},
		err: map[string]error{
			"https://web.archive.org/web/20240101120000id_/https://example.com/x": &fetch.Error{Status: 403, Msg: "blocked: private address 10.0.0.5"},
		},
	}
	_, err := Try(context.Background(), f, "https://example.com/x")
	if err == nil {
		t.Fatal("want error")
	}
	if !strings.Contains(err.Error(), "blocked:") && !strings.Contains(err.Error(), "no snapshot") && !strings.Contains(err.Error(), "private") {
		t.Fatalf("err %v", err)
	}
}

func TestWaybackHTTPRewrittenAnd204Skipped(t *testing.T) {
	const fixture = `{"url":"https://www.ft.com/","archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://www.ft.com/","timestamp":"20240101120000","status":"204"}}}`
	f := &fakeFetch{
		body: map[string][]byte{
			"https://archive.org/wayback/available": []byte(fixture),
			"https://archive.ph/newest/":            []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>"),
		},
		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
	}
	r, err := Try(context.Background(), f, "https://www.ft.com/")
	if err != nil {
		t.Fatal(err)
	}
	if r.Via != ViaToday {
		t.Fatalf("via %s hits %v", r.Via, f.hits)
	}
	for _, h := range f.hits {
		if strings.Contains(h, "web.archive.org") {
			t.Fatalf("fetched 204 snapshot: %v", f.hits)
		}
	}

	page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
	avail := []byte(`{"archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://example.com/x","timestamp":"20240101120000","status":"200"}}}`)
	f2 := &fakeFetch{
		body: map[string][]byte{
			"https://archive.org/wayback/available":                               avail,
			"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
		},
		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
	}
	r, err = Try(context.Background(), f2, "https://example.com/x")
	if err != nil {
		t.Fatal(err)
	}
	if r.Via != ViaWayback {
		t.Fatalf("via %s hits %v", r.Via, f2.hits)
	}
}

func TestMementoDatetime(t *testing.T) {
	h := make(http.Header)
	h.Set("Memento-Datetime", "Wed, 01 Jan 2020 12:00:00 GMT")
	got := mementoDatetime(&fetch.Resource{Header: h})
	if got != "2020-01-01T12:00:00Z" {
		t.Fatalf("got %q", got)
	}
	h.Set("Memento-Datetime", "Mon, 31 Dec 2029 00:00:00 GMT")
	if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
		t.Fatalf("future %q", got)
	}
	h.Set("Memento-Datetime", "not-a-date")
	if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
		t.Fatalf("garbage %q", got)
	}
}