sift/internal/recover/recover_test.go

package recover

import (
	"context"
	"encoding/json"
	"strings"
	"testing"
	"time"

	"git.wntrmute.dev/kyle/sift/internal/fetch"
)

type fakeFetch struct {
	body map[string][]byte
	ct   map[string]string
	err  map[string]error
	hits []string
}

func (f *fakeFetch) Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error) {
	f.hits = append(f.hits, rawURL)
	if e, ok := f.err[rawURL]; ok {
		return nil, e
	}
	for k, b := range f.body {
		if strings.HasPrefix(rawURL, k) || rawURL == k {
			ct := f.ct[k]
			if ct == "" {
				ct = "text/html"
			}
			return &fetch.Resource{
				URL: rawURL, FinalURL: rawURL, Status: 200,
				ContentType: ct, Body: b, FetchedAt: time.Now(),
			}, nil
		}
	}
	return nil, &fetch.Error{Status: 404, Msg: "missing " + rawURL}
}

func TestInsertID(t *testing.T) {
	in := "https://web.archive.org/web/20240101120000/https://example.com/x"
	got := InsertID(in)
	if !strings.Contains(got, "20240101120000id_/") {
		t.Fatalf("%s", got)
	}
}

func TestWaybackAvailable(t *testing.T) {
	snap := "https://web.archive.org/web/20240101120000/https://example.com/x"
	avail, _ := json.Marshal(map[string]any{
		"archived_snapshots": map[string]any{
			"closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"},
		},
	})
	page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
	f := &fakeFetch{
		body: map[string][]byte{
			"https://archive.org/wayback/available":                               avail,
			"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
		},
		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
	}
	r, err := Try(context.Background(), f, "https://example.com/x")
	if err != nil {
		t.Fatal(err)
	}
	if r.Via != ViaWayback || r.SnapshotAt != "20240101120000" {
		t.Fatalf("%+v", r)
	}
}

func TestWaybackRejectsForeignHost(t *testing.T) {
	avail, _ := json.Marshal(map[string]any{
		"archived_snapshots": map[string]any{
			"closest": map[string]any{"available": true, "url": "https://evil.example/x", "timestamp": "20240101120000"},
		},
	})
	f := &fakeFetch{
		body: map[string][]byte{"https://archive.org/wayback/available": avail},
		ct:   map[string]string{"https://archive.org/wayback/available": "application/json"},
	}
	_, err := Try(context.Background(), f, "https://example.com/x")
	if err == nil {
		t.Fatal("want reject")
	}
}

func TestWaybackUnavailableGoesToArchiveToday(t *testing.T) {
	avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
	page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
	f := &fakeFetch{
		body: map[string][]byte{
			"https://archive.org/wayback/available": avail,
			"https://archive.ph/newest/":            page,
		},
		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
	}
	r, err := Try(context.Background(), f, "https://example.com/x")
	if err != nil {
		t.Fatal(err)
	}
	if r.Via != ViaToday {
		t.Fatalf("via %s", r.Via)
	}
}

func TestFakeSuccessMetaRefresh(t *testing.T) {
	body := []byte(`<html><head><meta http-equiv="refresh" content="0;url=https://example.com/x"><title>Redirecting</title></head><body>` + strings.Repeat("x", 500) + `</body></html>`)
	if !fakeSuccess(&fetch.Resource{Body: body, FinalURL: "https://web.archive.org/foo"}, "https://example.com/x") {
		t.Fatal("want fake")
	}
}

func TestFakeSuccessJustAMoment(t *testing.T) {
	body := []byte(`<html><head><title>Just a moment...</title></head><body>` + strings.Repeat("x", 500) + `</body></html>`)
	if !fakeSuccess(&fetch.Resource{Body: body}, "https://example.com") {
		t.Fatal("want fake")
	}
}

func TestFakeSuccessTinyBody(t *testing.T) {
	if !fakeSuccess(&fetch.Resource{Body: []byte("short")}, "https://example.com") {
		t.Fatal("want fake")
	}
}

func TestArchiveTodayRotateOn429(t *testing.T) {
	page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
	avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
	f := &fakeFetch{
		body: map[string][]byte{
			"https://archive.org/wayback/available": avail,
			"https://archive.md/newest/":            page,
		},
		ct:  map[string]string{"https://archive.org/wayback/available": "application/json"},
		err: map[string]error{"https://archive.ph/newest/https://example.com/x": &fetch.Error{Status: 429, Msg: "busy"}},
	}
	r, err := Try(context.Background(), f, "https://example.com/x")
	if err != nil {
		t.Fatal(err)
	}
	if r.Via != ViaToday {
		t.Fatalf("via %s hits %v", r.Via, f.hits)
	}
}

func TestRedirectToPrivateBlocked(t *testing.T) {
	snap := "https://web.archive.org/web/20240101120000/https://example.com/x"
	avail, _ := json.Marshal(map[string]any{
		"archived_snapshots": map[string]any{
			"closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"},
		},
	})
	f := &fakeFetch{
		body: map[string][]byte{"https://archive.org/wayback/available": avail},
		ct:   map[string]string{"https://archive.org/wayback/available": "application/json"},
		err: map[string]error{
			"https://web.archive.org/web/20240101120000id_/https://example.com/x": &fetch.Error{Status: 403, Msg: "blocked: private address 10.0.0.5"},
		},
	}
	_, err := Try(context.Background(), f, "https://example.com/x")
	if err == nil {
		t.Fatal("want error")
	}
	if !strings.Contains(err.Error(), "blocked:") && !strings.Contains(err.Error(), "no snapshot") && !strings.Contains(err.Error(), "private") {
		t.Fatalf("err %v", err)
	}
}