package api import ( "context" "errors" "fmt" "time" "unicode/utf8" "git.wntrmute.dev/kyle/sift/internal/auth" "git.wntrmute.dev/kyle/sift/internal/batch" "git.wntrmute.dev/kyle/sift/internal/clean" "git.wntrmute.dev/kyle/sift/internal/digest" "git.wntrmute.dev/kyle/sift/internal/fetch" "git.wntrmute.dev/kyle/sift/internal/fleetctx" "git.wntrmute.dev/kyle/sift/internal/pdfdoc" "git.wntrmute.dev/kyle/sift/internal/recover" ) // fetchTarget runs one target's per-target pipeline: fetch → recover → PDF render or clean → // question limit → digest → context → log line → envelope. It is the single body both /v1/fetch // and the batch worker drive, so it must not change what v1fetch returned. func (s *Server) fetchTarget(ctx context.Context, c *auth.Caller, t batch.Target) (batch.Result, error) { q := t.Options.Question if len([]rune(q)) > 500 { rns := []rune(q) q = string(rns[:500]) } mode := t.Options.Mode if mode == "" { if q != "" { mode = "digest" } else { mode = "clean" } } if mode != "clean" && mode != "raw" && mode != "digest" { return batch.Result{}, &fetch.Error{Status: 400, Msg: "invalid url"} } wantCtx := q != "" if t.Options.Context != nil { wantCtx = *t.Options.Context } t0 := time.Now() onlyMain := true res, cl, doc, ferr := s.doFetch(ctx, c, t.URL, t.Options.JQ, onlyMain) var snap *recover.Result if ferr != nil && t.Options.Recover && recoverWorthy(ferr) { sr, rerr := recover.Try(ctx, s.Fetch, t.URL) if rerr == nil && sr != nil && sr.Res != nil { res, cl, doc, ferr = s.cleanFetched(sr.Res, t.Options.JQ, onlyMain) snap = sr } } if ferr != nil { return batch.Result{}, ferr } fetchMS := time.Since(t0).Milliseconds() if doc == nil && (t.Options.Pages != "" || t.Options.Grep != "") { return batch.Result{}, &fetch.Error{Status: 400, Msg: "pages and grep apply to PDFs only"} } var docLen, pdfPages int var content string var pdfMap map[string]any if doc != nil { pdfPages = doc.Pages if mode == "raw" { return batch.Result{}, &fetch.Error{Status: 400, Msg: "raw mode does not apply to PDFs"} } rp, err := s.renderPDF(doc, pdfdoc.Options{Pages: t.Options.Pages, Grep: t.Options.Grep, OutCap: s.OutCap}) if err != nil { if errors.Is(err, pdfdoc.ErrBadPages) || errors.Is(err, pdfdoc.ErrBadGrep) { return batch.Result{}, &fetch.Error{Status: 400, Msg: err.Error()} } return batch.Result{}, err } full := pdfdoc.Full(doc) docLen = utf8.RuneCountInString(full) if mode == "digest" { content = full } else { content = rp.Content } pdfMap = map[string]any{ "pages": doc.Pages, "title": doc.Title, "outline": doc.Outline, "extracted": len(doc.Text), "first": rp.First, "last": rp.Last, "map": rp.Map, "grep_hits": rp.GrepHits, "note": rp.Note, } } else { content = cl.Markdown if cl.Kind == "json" { content = cl.HTML } if mode == "raw" { content = string(res.Body) if len(content) > s.OutCap { content = clean.TruncateUTF8(content, s.OutCap) cl.Trunc = true cl.Warning = "truncated to output cap" } } } warn := []string{} if cl.Warning != "" { warn = append(warn, cl.Warning) } var digMeta map[string]any var digestMS int64 if mode == "digest" { if doc != nil { n := len(digest.Split(pdfdoc.Full(doc), digest.ChunkTokens)) limit := s.PDFMaxQuestionChunks if limit == 0 { limit = DefaultPDFMaxQuestionChunks } if n > limit { return batch.Result{}, &fetch.Error{Status: 422, Msg: fmt.Sprintf("document too long for question (%d chunks); use grep or pages", n)} } } if s.Digest == nil { warn = append(warn, "digest fell back to clean: unset") mode = "clean" content, warn = capDigestFallback(content, warn) digMeta = map[string]any{"fallback": "unset"} } else { dr := s.Digest.Run(ctx, content, q, res.FinalURL, s.callerClass(c)) digestMS = dr.DigestMS if !dr.Cached { s.Prom.ObserveDigest(time.Duration(dr.DigestMS) * time.Millisecond) } if dr.Fallback != "" { warn = append(warn, "digest fell back to clean: "+dr.Fallback) mode = "clean" content, warn = capDigestFallback(content, warn) digMeta = map[string]any{"fallback": dr.Fallback, "model": dr.Model} } else { content = dr.Text digMeta = map[string]any{ "model": dr.Model, "chunks_used": dr.ChunksUsed, "chunks_total": dr.ChunksTotal, "retrieval": dr.Retrieval, "embedded": dr.Embedded, "fallback": false, "cached": dr.Cached, } } } } var ctxItems []fleetctx.Item var ctxMS int64 if wantCtx && s.Ctx != nil { ct0 := time.Now() query := fleetctx.Query(q, cl.Title, headings(cl.Markdown)) items, cw := s.Ctx.Search(ctx, c.Workspace, c.Name, query) ctxMS = time.Since(ct0).Milliseconds() if cw != "" { warn = append(warn, cw) } ctxItems = items } qLen := len(q) if qLen > 80 { qLen = 80 } fb, _ := "", false if digMeta != nil { if s, ok := digMeta["fallback"].(string); ok { fb = s } } s.Log.Info("fetch", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "mode", mode, "q_len", qLen, "raw_tokens", len(res.Body)/4, "returned_tokens", len(content)/4, "pdf_pages", pdfPages, "digest_ms", digestMS, "fallback", fb) src := map[string]any{ "url": res.URL, "final_url": res.FinalURL, "status": res.Status, "content_type": res.ContentType, "fetched_at": res.FetchedAt.Format(time.RFC3339), "hash": res.SHA256(), "bytes": len(res.Body), "cached": len(res.Body) == 0, "kind": "live", } if snap != nil { src["kind"] = "snapshot" src["via"] = string(snap.Via) src["original_url"] = snap.OriginalURL if snap.SnapshotAt != "" { src["snapshot_at"] = snap.SnapshotAt } else { src["snapshot_at"] = nil warn = append(warn, "snapshot date unknown") } } m := map[string]any{ "mode": mode, "untrusted": true, "source": src, "warnings": warn, "sizes": map[string]int{ "raw_tokens": docLen / 4, "clean_tokens": digest.EstTokens(cl.Markdown), "returned_tokens": digest.EstTokens(content), }, "timings": map[string]int64{"fetch_ms": fetchMS, "digest_ms": digestMS, "context_ms": ctxMS}, } if pdfMap != nil { m["pdf"] = pdfMap } if digMeta != nil { m["digest"] = digMeta } if wantCtx { m["context"] = ctxItems } return batch.Result{Content: content, Meta: m}, nil }