digest: cache chunk embeddings; long documents answer from BM25 while embedding in the background

Cached chunk embeddings change what a long document answers with: BM25 first,
embedding uncached chunks (or the whole document) in the background so the next
question on the same doc scores hybrid. selectText now returns its mode and the
number of chunks that had a vector at answer-build time; the digest cache stores
only hybrid answers, never a provisional bm25 one. The background worker batches
through a Dixie gate and WaitBackground polls for true idle.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

internal/digest/digest.go

@@ -33,6 +33,8 @@ const (
 	ChunkTokens    = 1200
 	EmbedBatch     = 16
 	EmbedMaxRunes  = 3000
+	EmbedSyncMax   = 48
+	MaxDocChunks   = 5000
 	DefaultPermit  = 2
 	DefaultQDepth  = 8
 	DefaultQWait   = 20 * time.Second
@@ -191,6 +193,9 @@ type Engine struct {
 	Log        *slog.Logger
 	Cache      *dcache.DB
 
+	bgOnce sync.Once
+	bg     *bgWorker
+
 	mu       sync.Mutex
 	Calls    uint64
 	Fallback map[string]uint64
@@ -205,6 +210,8 @@ type Result struct {
 	DigestMS    int64
 	ReturnedTok int
 	Cached      bool
+	Retrieval   string // "whole", "head", "hybrid", "bm25" or "cached"
+	Embedded    int    // chunks that had an embedding when the answer was built
 }
 
 func (e *Engine) timeout() time.Duration {
@@ -305,6 +312,7 @@ func (e *Engine) Run(ctx context.Context, cleaned, question, finalURL, callerCla
 			r.ChunksUsed = h.ChunksUsed
 			r.ReturnedTok = EstTokens(h.Text)
 			r.Cached = true
+			r.Retrieval = "cached"
 			r.DigestMS = time.Since(start).Milliseconds()
 			return r
 		}
@@ -327,8 +335,10 @@ func (e *Engine) Run(ctx context.Context, cleaned, question, finalURL, callerCla
 	dctx, cancel := context.WithTimeout(ctx, e.timeout())
 	defer cancel()
 
-	body, nUsed, nTotal, err := e.selectText(dctx, cleaned, question)
+	body, nUsed, nTotal, mode, embedded, err := e.selectText(dctx, cleaned, question, contentHash, callerClass)
 	r.ChunksUsed, r.ChunksTotal = nUsed, nTotal
+	r.Retrieval = mode
+	r.Embedded = embedded
 	if err != nil {
 		e.logFallback("embed", err)
 		r.Fallback = "embed"
@@ -359,72 +369,26 @@ func (e *Engine) Run(ctx context.Context, cleaned, question, finalURL, callerCla
 	}
 	r.Text = text
 	r.ReturnedTok = EstTokens(text)
-	if e != nil && e.Cache != nil {
+	// A bm25 answer is provisional: once the embeddings exist the same question
+	// must get a hybrid answer, so it does not go in the digest cache.
+	if e != nil && e.Cache != nil && mode != "bm25" {
 		e.Cache.PutDigest(dkey, text, r.ChunksUsed)
 	}
 	return r
 }
 
-func (e *Engine) selectText(ctx context.Context, cleaned, question string) (string, int, int, error) {
+func (e *Engine) selectText(ctx context.Context, cleaned, question, contentHash, callerClass string) (string, int, int, string, int, error) {
 	if EstTokens(cleaned) <= SmallDoc {
-		return cleaned, 1, 1, nil
+		return cleaned, 1, 1, "whole", 0, nil
 	}
-	chunks := Split(cleaned, ChunkTokens)
+	chunks := SplitPages(cleaned, ChunkTokens)
 	if len(chunks) == 0 {
-		return cleaned, 1, 1, nil
+		return cleaned, 1, 1, "whole", 0, nil
 	}
 	if question == "" {
-		return joinBudget(chunks, SmallDoc), min(len(chunks), countBudget(chunks, SmallDoc)), len(chunks), nil
+		return joinBudget(chunks, SmallDoc), min(len(chunks), countBudget(chunks, SmallDoc)), len(chunks), "head", 0, nil
 	}
-	vecs, err := e.embed(ctx, append([]string{question}, capEmbed(chunks)...))
-	if err != nil || len(vecs) < 2 {
-		return joinBudget(chunks, SmallDoc), min(len(chunks), countBudget(chunks, SmallDoc)), len(chunks), err
-	}
-	q := vecs[0]
-	type scored struct {
-		i int
-		s float64
-	}
-	var ss []scored
-	for i, c := range chunks {
-		if i+1 >= len(vecs) {
-			break
-		}
-		ss = append(ss, scored{i: i, s: 0.7*cosine(q, vecs[i+1]) + 0.3*lexicalOverlap(question, c)})
-	}
-	picked := map[int]struct{}{}
-	used := 0
-	if len(chunks) > 0 {
-		picked[0] = struct{}{}
-		used = EstTokens(chunks[0])
-	}
-	for len(picked) < len(ss) {
-		best, bi := -1.0, -1
-		for _, x := range ss {
-			if _, ok := picked[x.i]; ok {
-				continue
-			}
-			if x.s > best {
-				best, bi = x.s, x.i
-			}
-		}
-		if bi < 0 {
-			break
-		}
-		need := EstTokens(chunks[bi])
-		if used+need > SmallDoc && used > 0 {
-			break
-		}
-		picked[bi] = struct{}{}
-		used += need
-	}
-	var ordered []string
-	for i := range chunks {
-		if _, ok := picked[i]; ok {
-			ordered = append(ordered, chunks[i])
-		}
-	}
-	return strings.Join(ordered, "\n\n"), len(ordered), len(chunks), nil
+	return e.hybridOrBM25(ctx, chunks, question, contentHash, callerClass)
 }
 
 func capEmbed(chunks []string) []string {