p2.1: honcho-embed, wayback http/204, memento, keys owner
Embed requests send dixie.embed_model (default honcho-embed). Wayback accepts http://web.archive.org, rewrites to https, skips non-200 closest.status. Memento-Datetime is parsed as an HTTP date and future/garbage values become empty snapshot_at. keys add writes via temp+fchown/chmod/rename so root does not clobber keys.toml. Digest fallback is capped at the 6k-token budget.
This commit is contained in:
@@ -6,9 +6,11 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/fetch"
|
||||
)
|
||||
@@ -61,9 +63,10 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
|
||||
var env struct {
|
||||
ArchivedSnapshots struct {
|
||||
Closest struct {
|
||||
Available bool `json:"available"`
|
||||
URL string `json:"url"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Available bool `json:"available"`
|
||||
URL string `json:"url"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Status json.RawMessage `json:"status"`
|
||||
} `json:"closest"`
|
||||
} `json:"archived_snapshots"`
|
||||
}
|
||||
@@ -74,11 +77,15 @@ func wayback(ctx context.Context, f Fetcher, original string) (*Result, error) {
|
||||
if !c.Available || c.URL == "" {
|
||||
return nil, fmt.Errorf("wayback: unavailable")
|
||||
}
|
||||
if !waybackStatusOK(c.Status) {
|
||||
return nil, fmt.Errorf("wayback: status %s", strings.TrimSpace(string(c.Status)))
|
||||
}
|
||||
u, err := url.Parse(c.URL)
|
||||
if err != nil || u.Scheme != "https" || strings.ToLower(u.Hostname()) != "web.archive.org" {
|
||||
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || strings.ToLower(u.Hostname()) != "web.archive.org" {
|
||||
return nil, fmt.Errorf("wayback: bad snapshot url")
|
||||
}
|
||||
snapURL := insertID(c.URL)
|
||||
u.Scheme = "https"
|
||||
snapURL := insertID(u.String())
|
||||
page, err := f.Fetch(ctx, snapURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -170,7 +177,26 @@ func mementoDatetime(res *fetch.Resource) string {
|
||||
if res == nil || res.Header == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(res.Header.Get("Memento-Datetime"))
|
||||
raw := strings.TrimSpace(res.Header.Get("Memento-Datetime"))
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
t, err := http.ParseTime(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if t.After(time.Now().Add(24 * time.Hour)) {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func waybackStatusOK(raw json.RawMessage) bool {
|
||||
if len(bytesTrim(raw)) == 0 || string(raw) == "null" {
|
||||
return true
|
||||
}
|
||||
s := strings.Trim(strings.TrimSpace(string(raw)), `"`)
|
||||
return s == "200"
|
||||
}
|
||||
|
||||
func hostOf(raw string) string {
|
||||
|
||||
Reference in New Issue
Block a user