p2.1: honcho-embed, wayback http/204, memento, keys owner
Embed requests send dixie.embed_model (default honcho-embed). Wayback accepts http://web.archive.org, rewrites to https, skips non-200 closest.status. Memento-Datetime is parsed as an HTTP date and future/garbage values become empty snapshot_at. keys add writes via temp+fchown/chmod/rename so root does not clobber keys.toml. Digest fallback is capped at the 6k-token budget.
This commit is contained in:
@@ -3,6 +3,7 @@ package recover
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -166,3 +167,60 @@ func TestRedirectToPrivateBlocked(t *testing.T) {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaybackHTTPRewrittenAnd204Skipped(t *testing.T) {
|
||||
const fixture = `{"url":"https://www.ft.com/","archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://www.ft.com/","timestamp":"20240101120000","status":"204"}}}`
|
||||
f := &fakeFetch{
|
||||
body: map[string][]byte{
|
||||
"https://archive.org/wayback/available": []byte(fixture),
|
||||
"https://archive.ph/newest/": []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>"),
|
||||
},
|
||||
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
|
||||
}
|
||||
r, err := Try(context.Background(), f, "https://www.ft.com/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Via != ViaToday {
|
||||
t.Fatalf("via %s hits %v", r.Via, f.hits)
|
||||
}
|
||||
for _, h := range f.hits {
|
||||
if strings.Contains(h, "web.archive.org") {
|
||||
t.Fatalf("fetched 204 snapshot: %v", f.hits)
|
||||
}
|
||||
}
|
||||
|
||||
page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
|
||||
avail := []byte(`{"archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://example.com/x","timestamp":"20240101120000","status":"200"}}}`)
|
||||
f2 := &fakeFetch{
|
||||
body: map[string][]byte{
|
||||
"https://archive.org/wayback/available": avail,
|
||||
"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
|
||||
},
|
||||
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
|
||||
}
|
||||
r, err = Try(context.Background(), f2, "https://example.com/x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Via != ViaWayback {
|
||||
t.Fatalf("via %s hits %v", r.Via, f2.hits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMementoDatetime(t *testing.T) {
|
||||
h := make(http.Header)
|
||||
h.Set("Memento-Datetime", "Wed, 01 Jan 2020 12:00:00 GMT")
|
||||
got := mementoDatetime(&fetch.Resource{Header: h})
|
||||
if got != "2020-01-01T12:00:00Z" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
h.Set("Memento-Datetime", "Mon, 31 Dec 2029 00:00:00 GMT")
|
||||
if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
|
||||
t.Fatalf("future %q", got)
|
||||
}
|
||||
h.Set("Memento-Datetime", "not-a-date")
|
||||
if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
|
||||
t.Fatalf("garbage %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user