p2.1: honcho-embed, wayback http/204, memento, keys owner

Embed requests send dixie.embed_model (default honcho-embed).
Wayback accepts http://web.archive.org, rewrites to https, skips
non-200 closest.status. Memento-Datetime is parsed as an HTTP date
and future/garbage values become empty snapshot_at. keys add writes
via temp+fchown/chmod/rename so root does not clobber keys.toml.
Digest fallback is capped at the 6k-token budget.

internal/recover/recover.go

@@ -6,9 +6,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
+	"time"
 
 	"git.wntrmute.dev/kyle/sift/internal/fetch"
 )
@@ -61,9 +63,10 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
 	var env struct {
 		ArchivedSnapshots struct {
 			Closest struct {
-				Available bool   `json:"available"`
-				URL       string `json:"url"`
-				Timestamp string `json:"timestamp"`
+				Available bool            `json:"available"`
+				URL       string          `json:"url"`
+				Timestamp string          `json:"timestamp"`
+				Status    json.RawMessage `json:"status"`
 			} `json:"closest"`
 		} `json:"archived_snapshots"`
 	}
@@ -74,11 +77,15 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
 	if !c.Available || c.URL == "" {
 		return nil, fmt.Errorf("wayback: unavailable")
 	}
+	if !waybackStatusOK(c.Status) {
+		return nil, fmt.Errorf("wayback: status %s", strings.TrimSpace(string(c.Status)))
+	}
 	u, err := url.Parse(c.URL)
-	if err != nil || u.Scheme != "https" || strings.ToLower(u.Hostname()) != "web.archive.org" {
+	if err != nil || (u.Scheme != "https" && u.Scheme != "http") || strings.ToLower(u.Hostname()) != "web.archive.org" {
 		return nil, fmt.Errorf("wayback: bad snapshot url")
 	}
-	snapURL := insertID(c.URL)
+	u.Scheme = "https"
+	snapURL := insertID(u.String())
 	page, err := f.Fetch(ctx, snapURL)
 	if err != nil {
 		return nil, err
@@ -170,7 +177,26 @@ func mementoDatetime(res *fetch.Resource) string {
 	if res == nil || res.Header == nil {
 		return ""
 	}
-	return strings.TrimSpace(res.Header.Get("Memento-Datetime"))
+	raw := strings.TrimSpace(res.Header.Get("Memento-Datetime"))
+	if raw == "" {
+		return ""
+	}
+	t, err := http.ParseTime(raw)
+	if err != nil {
+		return ""
+	}
+	if t.After(time.Now().Add(24 * time.Hour)) {
+		return ""
+	}
+	return t.UTC().Format(time.RFC3339)
+}
+
+func waybackStatusOK(raw json.RawMessage) bool {
+	if len(bytesTrim(raw)) == 0 || string(raw) == "null" {
+		return true
+	}
+	s := strings.Trim(strings.TrimSpace(string(raw)), `"`)
+	return s == "200"
 }
 
 func hostOf(raw string) string {

internal/recover/recover_test.go

@@ -3,6 +3,7 @@ package recover
 import (
 	"context"
 	"encoding/json"
+	"net/http"
 	"strings"
 	"testing"
 	"time"
@@ -166,3 +167,60 @@ func TestRedirectToPrivateBlocked(t *testing.T) {
 		t.Fatalf("err %v", err)
 	}
 }
+
+func TestWaybackHTTPRewrittenAnd204Skipped(t *testing.T) {
+	const fixture = `{"url":"https://www.ft.com/","archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://www.ft.com/","timestamp":"20240101120000","status":"204"}}}`
+	f := &fakeFetch{
+		body: map[string][]byte{
+			"https://archive.org/wayback/available": []byte(fixture),
+			"https://archive.ph/newest/":            []byte("<html><head><title>Saved</title></head><body>" + strings.Repeat("ok ", 200) + "</body></html>"),
+		},
+		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
+	}
+	r, err := Try(context.Background(), f, "https://www.ft.com/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.Via != ViaToday {
+		t.Fatalf("via %s hits %v", r.Via, f.hits)
+	}
+	for _, h := range f.hits {
+		if strings.Contains(h, "web.archive.org") {
+			t.Fatalf("fetched 204 snapshot: %v", f.hits)
+		}
+	}
+
+	page := []byte("<html><head><title>Article</title></head><body>" + strings.Repeat("paragraph ", 80) + "</body></html>")
+	avail := []byte(`{"archived_snapshots":{"closest":{"available":true,"url":"http://web.archive.org/web/20240101120000/https://example.com/x","timestamp":"20240101120000","status":"200"}}}`)
+	f2 := &fakeFetch{
+		body: map[string][]byte{
+			"https://archive.org/wayback/available":                               avail,
+			"https://web.archive.org/web/20240101120000id_/https://example.com/x": page,
+		},
+		ct: map[string]string{"https://archive.org/wayback/available": "application/json"},
+	}
+	r, err = Try(context.Background(), f2, "https://example.com/x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.Via != ViaWayback {
+		t.Fatalf("via %s hits %v", r.Via, f2.hits)
+	}
+}
+
+func TestMementoDatetime(t *testing.T) {
+	h := make(http.Header)
+	h.Set("Memento-Datetime", "Wed, 01 Jan 2020 12:00:00 GMT")
+	got := mementoDatetime(&fetch.Resource{Header: h})
+	if got != "2020-01-01T12:00:00Z" {
+		t.Fatalf("got %q", got)
+	}
+	h.Set("Memento-Datetime", "Mon, 31 Dec 2029 00:00:00 GMT")
+	if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
+		t.Fatalf("future %q", got)
+	}
+	h.Set("Memento-Datetime", "not-a-date")
+	if got := mementoDatetime(&fetch.Resource{Header: h}); got != "" {
+		t.Fatalf("garbage %q", got)
+	}
+}