package recover import ( "context" "encoding/json" "net/http" "strings" "testing" "time" "git.wntrmute.dev/kyle/sift/internal/fetch" ) type fakeFetch struct { body map[string][]byte ct map[string]string err map[string]error hits []string } func (f *fakeFetch) Fetch(ctx context.Context, rawURL string) (*fetch.Resource, error) { f.hits = append(f.hits, rawURL) if e, ok := f.err[rawURL]; ok { return nil, e } for k, b := range f.body { if strings.HasPrefix(rawURL, k) || rawURL == k { ct := f.ct[k] if ct == "" { ct = "text/html" } return &fetch.Resource{ URL: rawURL, FinalURL: rawURL, Status: 200, ContentType: ct, Body: b, FetchedAt: time.Now(), }, nil } } return nil, &fetch.Error{Status: 404, Msg: "missing " + rawURL} } func TestInsertID(t *testing.T) { in := "https://web.archive.org/web/20240101120000/https://example.com/x" got := InsertID(in) if !strings.Contains(got, "20240101120000id_/") { t.Fatalf("%s", got) } } func TestWaybackAvailable(t *testing.T) { snap := "https://web.archive.org/web/20240101120000/https://example.com/x" avail, _ := json.Marshal(map[string]any{ "archived_snapshots": map[string]any{ "closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"}, }, }) page := []byte("Article" + strings.Repeat("paragraph ", 80) + "") f := &fakeFetch{ body: map[string][]byte{ "https://archive.org/wayback/available": avail, "https://web.archive.org/web/20240101120000id_/https://example.com/x": page, }, ct: map[string]string{"https://archive.org/wayback/available": "application/json"}, } r, err := Try(context.Background(), f, "https://example.com/x") if err != nil { t.Fatal(err) } if r.Via != ViaWayback || r.SnapshotAt != "20240101120000" { t.Fatalf("%+v", r) } } func TestWaybackRejectsForeignHost(t *testing.T) { avail, _ := json.Marshal(map[string]any{ "archived_snapshots": map[string]any{ "closest": map[string]any{"available": true, "url": "https://evil.example/x", "timestamp": "20240101120000"}, }, }) f := &fakeFetch{ body: map[string][]byte{"https://archive.org/wayback/available": avail}, ct: map[string]string{"https://archive.org/wayback/available": "application/json"}, } _, err := Try(context.Background(), f, "https://example.com/x") if err == nil { t.Fatal("want reject") } } func TestWaybackUnavailableGoesToArchiveToday(t *testing.T) { avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}}) page := []byte("Saved" + strings.Repeat("ok ", 200) + "") f := &fakeFetch{ body: map[string][]byte{ "https://archive.org/wayback/available": avail, "https://archive.ph/newest/": page, }, ct: map[string]string{"https://archive.org/wayback/available": "application/json"}, } r, err := Try(context.Background(), f, "https://example.com/x") if err != nil { t.Fatal(err) } if r.Via != ViaToday { t.Fatalf("via %s", r.Via) } } func TestFakeSuccessMetaRefresh(t *testing.T) { body := []byte(`Redirecting` + strings.Repeat("x", 500) + ``) if !fakeSuccess(&fetch.Resource{Body: body, FinalURL: "https://web.archive.org/foo"}, "https://example.com/x") { t.Fatal("want fake") } } func TestFakeSuccessJustAMoment(t *testing.T) { body := []byte(`Just a moment...` + strings.Repeat("x", 500) + ``) if !fakeSuccess(&fetch.Resource{Body: body}, "https://example.com") { t.Fatal("want fake") } } func TestFakeSuccessTinyBody(t *testing.T) { if !fakeSuccess(&fetch.Resource{Body: []byte("short")}, "https://example.com") { t.Fatal("want fake") } } func TestArchiveTodayRotateOn429(t *testing.T) { page := []byte("Saved" + strings.Repeat("ok ", 200) + "") avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}}) f := &fakeFetch{ body: map[string][]byte{ "https://archive.org/wayback/available": avail, "https://archive.md/newest/": page, }, ct: map[string]string{"https://archive.org/wayback/available": "application/json"}, err: map[string]error{"https://archive.ph/newest/https://example.com/x": &fetch.Error{Status: 429, Msg: "busy"}}, } r, err := Try(context.Background(), f, "https://example.com/x") if err != nil { t.Fatal(err) } if r.Via != ViaToday { t.Fatalf("via %s hits %v", r.Via, f.hits) } } func TestRedirectToPrivateBlocked(t *testing.T) { snap := "https://web.archive.org/web/20240101120000/https://example.com/x" avail, _ := json.Marshal(map[string]any{ "archived_snapshots": map[string]any{ "closest": map[string]any{"available": true, "url": snap, "timestamp": "20240101120000"}, }, }) f := &fakeFetch{ body: map[string][]byte{"https://archive.org/wayback/available": avail}, ct: map[string]string{"https://archive.org/wayback/available": "application/json"}, err: map[string]error{ "https://web.archive.org/web/20240101120000id_/https://example.com/x": &fetch.Error{Status: 403, Msg: "blocked: private address 10.0.0.5"}, }, } _, err := Try(context.Background(), f, "https://example.com/x") if err == nil { t.Fatal("want error") } if !strings.Contains(err.Error(), "blocked:") && !strings.Contains(err.Error(), "no snapshot") && !strings.Contains(err.Error(), "private") { t.Fatalf("err %v", err) } } func TestWaybackHTTPRewrittenAnd204Skipped(t *testing.T) { const fixture = `{"url":"https://www.ft.com/","archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://www.ft.com/","timestamp":"20240101120000","status":"204"}}}` f := &fakeFetch{ body: map[string][]byte{ "https://archive.org/wayback/available": []byte(fixture), "https://archive.ph/newest/": []byte("Saved" + strings.Repeat("ok ", 200) + ""), }, ct: map[string]string{"https://archive.org/wayback/available": "application/json"}, } r, err := Try(context.Background(), f, "https://www.ft.com/") if err != nil { t.Fatal(err) } if r.Via != ViaToday { t.Fatalf("via %s hits %v", r.Via, f.hits) } for _, h := range f.hits { if strings.Contains(h, "web.archive.org") { t.Fatalf("fetched 204 snapshot: %v", f.hits) } } page := []byte("Article" + strings.Repeat("paragraph ", 80) + "") avail := []byte(`{"archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://example.com/x","timestamp":"20240101120000","status":"200"}}}`) f2 := &fakeFetch{ body: map[string][]byte{ "https://archive.org/wayback/available": avail, "https://web.archive.org/web/20240101120000id_/https://example.com/x": page, }, ct: map[string]string{"https://archive.org/wayback/available": "application/json"}, } r, err = Try(context.Background(), f2, "https://example.com/x") if err != nil { t.Fatal(err) } if r.Via != ViaWayback { t.Fatalf("via %s hits %v", r.Via, f2.hits) } } func TestMementoDatetime(t *testing.T) { h := make(http.Header) h.Set("Memento-Datetime", "Wed, 01 Jan 2020 12:00:00 GMT") got := mementoDatetime(&fetch.Resource{Header: h}) if got != "2020-01-01T12:00:00Z" { t.Fatalf("got %q", got) } h.Set("Memento-Datetime", "Mon, 31 Dec 2029 00:00:00 GMT") if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" { t.Fatalf("future %q", got) } h.Set("Memento-Datetime", "not-a-date") if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" { t.Fatalf("garbage %q", got) } }