kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

p2: digest with in-sift FIFO, SQLite caches, Honcho context, recover

Browse Source 
Dixie is capped (max_inflight 2, queue_depth 8, queue_wait 20s) and
never queued on llama-server. Streaming chat, thinking off, 45s
fallback to clean. Digest and chunk-embedding caches are
content-addressed SQLite (Honcho context is not cached). Recover
uses the guarded Fetcher. guardian-star is refused in code.
This commit is contained in:
kyle
2026-09-22 23:54:49 -07:00
parent 505e113b64
commit 2f9d3d97e6
2023 changed files with 3815943 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/api.go
+165 -21
View File
@@ -16,8 +16,11 @@ import (
"git.wntrmute.dev/kyle/sift/internal/auth"
"git.wntrmute.dev/kyle/sift/internal/clean"
"git.wntrmute.dev/kyle/sift/internal/digest"
"git.wntrmute.dev/kyle/sift/internal/fetch"
"git.wntrmute.dev/kyle/sift/internal/fleetctx"
"git.wntrmute.dev/kyle/sift/internal/guard"
"git.wntrmute.dev/kyle/sift/internal/recover"
"git.wntrmute.dev/kyle/sift/internal/selfip"
)
@@ -32,6 +35,8 @@ type Server struct {
Mux *http.ServeMux
OutCap int
SelfIP *selfip.Discoverer
Digest *digest.Engine
Ctx *fleetctx.Client
global chan struct{}
perMu sync.Mutex
@@ -123,6 +128,7 @@ type v1Req struct {
Mode string `json:"mode"`
Context *bool `json:"context"`
JQ string `json:"jq"`
Recover bool `json:"recover"`
}
func (s *Server) caller(w http.ResponseWriter, r *http.Request) *auth.Caller {
@@ -223,27 +229,48 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
writeErr(w, 400, "invalid url", "")
return
}
if req.Mode == "digest" || req.Question != "" || (req.Context != nil && *req.Context) {
writeErr(w, 400, "not implemented until P2", "")
return
q := req.Question
if len([]rune(q)) > 500 {
rns := []rune(q)
q = string(rns[:500])
}
mode := req.Mode
if mode == "" {
mode = "clean"
if q != "" {
mode = "digest"
} else {
mode = "clean"
}
}
if mode != "clean" && mode != "raw" {
if mode != "clean" && mode != "raw" && mode != "digest" {
writeErr(w, 400, "invalid url", "")
return
}
wantCtx := q != ""
if req.Context != nil {
wantCtx = *req.Context
}
t0 := time.Now()
onlyMain := true
res, cl, ferr := s.doFetch(r, c, req.URL, req.JQ, onlyMain)
var snap *recover.Result
if ferr != nil && req.Recover && recoverWorthy(ferr) {
sr, rerr := recover.Try(r.Context(), s.Fetch, req.URL)
if rerr == nil && sr != nil && sr.Res != nil {
res, cl, ferr = s.cleanFetched(sr.Res, req.JQ, onlyMain)
snap = sr
}
}
if ferr != nil {
s.writeFetchErr(w, c, req.URL, ferr)
return
}
fetchMS := time.Since(t0).Milliseconds()
content := cl.Markdown
if cl.Kind == "json" {
content = cl.HTML // pretty json without fence
content = cl.HTML
}
if mode == "raw" {
content = string(res.Body)
@@ -257,27 +284,144 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if cl.Warning != "" {
warn = append(warn, cl.Warning)
}
s.Log.Info("fetch", "caller", c.Name, "host", hostOf(res.FinalURL), "status", res.Status, "bytes", len(res.Body))
writeJSON(w, 200, map[string]any{
var digMeta map[string]any
var digestMS int64
if mode == "digest" {
if s.Digest == nil {
warn = append(warn, "digest fell back to clean: unset")
mode = "clean"
digMeta = map[string]any{"fallback": "unset"}
} else {
dr := s.Digest.Run(r.Context(), content, q, res.FinalURL)
digestMS = dr.DigestMS
if dr.Fallback != "" {
warn = append(warn, "digest fell back to clean: "+dr.Fallback)
mode = "clean"
content = clean.TruncateUTF8(content, s.OutCap)
digMeta = map[string]any{"fallback": dr.Fallback, "model": dr.Model}
} else {
content = dr.Text
digMeta = map[string]any{
"model": dr.Model,
"chunks_used": dr.ChunksUsed,
"chunks_total": dr.ChunksTotal,
"fallback": false,
"cached": dr.Cached,
}
}
}
}
var ctxItems []fleetctx.Item
var ctxMS int64
if wantCtx && s.Ctx != nil {
ct0 := time.Now()
query := fleetctx.Query(q, cl.Title, headings(cl.Markdown))
items, cw := s.Ctx.Search(r.Context(), c.Workspace, query)
ctxMS = time.Since(ct0).Milliseconds()
if cw != "" {
warn = append(warn, cw)
}
ctxItems = items
}
qLen := len(q)
if qLen > 80 {
qLen = 80
}
fb, _ := "", false
if digMeta != nil {
if s, ok := digMeta["fallback"].(string); ok {
fb = s
}
}
s.Log.Info("fetch", "caller", c.Name, "host", hostOf(res.FinalURL), "mode", mode,
"q_len", qLen, "raw_tokens", len(res.Body)/4, "returned_tokens", len(content)/4,
"digest_ms", digestMS, "fallback", fb)
src := map[string]any{
"url": res.URL,
"final_url": res.FinalURL,
"status": res.Status,
"content_type": res.ContentType,
"fetched_at": res.FetchedAt.Format(time.RFC3339),
"hash": res.SHA256(),
"bytes": len(res.Body),
"cached": false,
"kind": "live",
}
if snap != nil {
src["kind"] = "snapshot"
src["snapshot_at"] = snap.SnapshotAt
src["via"] = string(snap.Via)
src["original_url"] = snap.OriginalURL
}
out := map[string]any{
"content": content,
"mode": mode,
"untrusted": true,
"source": map[string]any{
"url": res.URL,
"final_url": res.FinalURL,
"status": res.Status,
"content_type": res.ContentType,
"fetched_at": res.FetchedAt.Format(time.RFC3339),
"hash": res.SHA256(),
"bytes": len(res.Body),
"cached": false,
},
"warnings": warn,
"source": src,
"warnings": warn,
"sizes": map[string]int{
"raw_tokens": len(res.Body) / 4,
"returned_tokens": len(content) / 4,
"clean_tokens": digest.EstTokens(cl.Markdown),
"returned_tokens": digest.EstTokens(content),
},
})
"timings": map[string]int64{"fetch_ms": fetchMS, "digest_ms": digestMS, "context_ms": ctxMS},
}
if digMeta != nil {
out["digest"] = digMeta
}
if wantCtx {
out["context"] = ctxItems
}
writeJSON(w, 200, out)
}
func recoverWorthy(err error) bool {
var fe *fetch.Error
if !errors.As(err, &fe) {
return false
}
if strings.Contains(fe.Msg, "blocked:") {
return false
}
return fe.Hint == "browser" || fe.Status == 422 || fe.Status == 429 || fe.Status == 504
}
func headings(md string) []string {
var h []string
for _, line := range strings.Split(md, "\n") {
if strings.HasPrefix(line, "#") {
h = append(h, strings.TrimLeft(line, "# "))
}
}
return h
}
func (s *Server) cleanFetched(res *fetch.Resource, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, error) {
kind := clean.MediaKind(res.ContentType)
if kind == "sniff" {
kind = clean.Sniff(res.Body)
}
if kind == "other" {
return nil, nil, &fetch.Error{Status: 415, Hint: "browser", Msg: "unsupported content type"}
}
var cl *clean.Result
var cerr error
switch kind {
case "html":
cl, cerr = clean.HTML(res, onlyMain, s.OutCap)
case "json":
cl, cerr = clean.JSON(res, jq, s.OutCap)
default:
cl, cerr = clean.Text(res, s.OutCap)
}
if cerr != nil {
return nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
}
return res, cl, nil
}
func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, error) {
internal/api/api_test.go
+13 -4
View File
@@ -138,12 +138,21 @@ func TestSkipTLSVerificationIgnored(t *testing.T) {
}
}
func TestP2Rejected(t *testing.T) {
s, key := setupAPI(t, nil)
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": "http://example.com/", "mode": "digest"})
if rec.Code != 400 || !strings.Contains(rec.Body.String(), "P2") {
func TestDigestFallbackWhenUnset(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("<html><body><p>hello</p></body></html>"))
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "mode": "digest", "question": "what"})
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "fallback") {
t.Fatalf("want fallback: %s", rec.Body.String())
}
}
func TestNever502(t *testing.T) {