p2: fence document tags, atomic fleetctx, harden workspace
Neutralise </document and <document in page text. Counters are atomic.Uint64. Workspace names are [A-Za-z0-9_-] and PathEscaped; foreign workspace_id rows are dropped. Wayback closest.url must be https://web.archive.org. archive.today snapshot_at is Memento-Datetime or null.
This commit is contained in:
@@ -9,7 +9,6 @@ import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/fetch"
|
||||
)
|
||||
@@ -75,6 +74,10 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
|
||||
if !c.Available || c.URL == "" {
|
||||
return nil, fmt.Errorf("wayback: unavailable")
|
||||
}
|
||||
u, err := url.Parse(c.URL)
|
||||
if err != nil || u.Scheme != "https" || strings.ToLower(u.Hostname()) != "web.archive.org" {
|
||||
return nil, fmt.Errorf("wayback: bad snapshot url")
|
||||
}
|
||||
snapURL := insertID(c.URL)
|
||||
page, err := f.Fetch(ctx, snapURL)
|
||||
if err != nil {
|
||||
@@ -122,7 +125,8 @@ func archiveToday(ctx context.Context, f Fetcher, original string) (*Result, err
|
||||
last = fmt.Errorf("archive.today: fake success")
|
||||
continue
|
||||
}
|
||||
return &Result{Res: res, Via: ViaToday, SnapshotAt: res.FetchedAt.UTC().Format(time.RFC3339), OriginalURL: original}, nil
|
||||
at := mementoDatetime(res)
|
||||
return &Result{Res: res, Via: ViaToday, SnapshotAt: at, OriginalURL: original}, nil
|
||||
}
|
||||
if last == nil {
|
||||
last = fmt.Errorf("archive.today: exhausted")
|
||||
@@ -162,6 +166,13 @@ func bytesTrim(b []byte) []byte {
|
||||
return []byte(strings.TrimSpace(string(b)))
|
||||
}
|
||||
|
||||
func mementoDatetime(res *fetch.Resource) string {
|
||||
if res == nil || res.Header == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(res.Header.Get("Memento-Datetime"))
|
||||
}
|
||||
|
||||
func hostOf(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -69,6 +69,22 @@ func TestWaybackAvailable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaybackRejectsForeignHost(t *testing.T) {
|
||||
avail, _ := json.Marshal(map[string]any{
|
||||
"archived_snapshots": map[string]any{
|
||||
"closest": map[string]any{"available": true, "url": "https://evil.example/x", "timestamp": "20240101120000"},
|
||||
},
|
||||
})
|
||||
f := &fakeFetch{
|
||||
body: map[string][]byte{"https://archive.org/wayback/available": avail},
|
||||
ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
|
||||
}
|
||||
_, err := Try(context.Background(), f, "https://example.com/x")
|
||||
if err == nil {
|
||||
t.Fatal("want reject")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaybackUnavailableGoesToArchiveToday(t *testing.T) {
|
||||
avail, _ := json.Marshal(map[string]any{"archived_snapshots": map[string]any{}})
|
||||
page := []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>")
|
||||
|
||||
Reference in New Issue
Block a user