api: questions on long PDFs are answered (retrieval in the digest meta); README and skill

The digest now handles long documents itself (task 02), so the API's refusal
limit is only a safety cap: DefaultPDFMaxQuestionChunks 200 -> 5000 (the 422
message text is unchanged). In v1fetch the digest-meta object gains "retrieval"
and "embedded" next to chunks_used/chunks_total, only when the digest ran
without a fallback. The PDF section explains a long-doc question is answered
first from BM25 while its chunks embed in the background and hybrid on later
questions; skill/SKILL.md gains one sentence. Given tests byte-identical.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

internal/api/api.go

@@ -569,6 +569,8 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
 					"model":        dr.Model,
 					"chunks_used":  dr.ChunksUsed,
 					"chunks_total": dr.ChunksTotal,
+					"retrieval":    dr.Retrieval,
+					"embedded":     dr.Embedded,
 					"fallback":     false,
 					"cached":       dr.Cached,
 				}

internal/api/pdf.go

@@ -20,9 +20,11 @@ import (
 const (
 	// DefaultPDFTTL is how long a cached PDF is served before it is revalidated.
 	DefaultPDFTTL = 24 * time.Hour
-	// DefaultPDFMaxQuestionChunks is the longest document (in digest chunks) a question is
-	// answered over; a longer one is refused in favour of grep or pages.
-	DefaultPDFMaxQuestionChunks = 200
+	// DefaultPDFMaxQuestionChunks is the safety cap on how long a document a question is
+	// answered over; beyond it the question is refused in favour of grep or pages. Longer
+	// documents are handled by retrieval (BM25 first, embedding their chunks in the
+	// background), so this only bounds the un-answerable.
+	DefaultPDFMaxQuestionChunks = 5000
 )
 
 // runner returns the configured extraction child, or a zero Runner that runs

internal/api/pdf_retrieval_test.go

@@ -0,0 +1,75 @@
+package api
+
+// P5b task 03: a question on a long PDF is answered (by BM25 first) instead of refused, and the
+// envelope says how the digest was built.
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/digest"
+	"git.wntrmute.dev/kyle/sift/internal/pdftest"
+)
+
+// fakeModel answers /embed with a fixed vector and /chat with a one-word SSE reply.
+func fakeModel(t *testing.T) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/embed", func(w http.ResponseWriter, r *http.Request) {
+		var body struct {
+			Input []string `json:"input"`
+		}
+		_ = json.NewDecoder(r.Body).Decode(&body)
+		var data []map[string]any
+		for i := range body.Input {
+			data = append(data, map[string]any{"index": i, "embedding": []float64{1, float64(i % 7)}})
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"data": data})
+	})
+	mux.HandleFunc("/chat", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream")
+		_, _ = w.Write([]byte(`data: {"choices":[{"delta":{"content":"answer (p. 3)"}}]}` + "\n\ndata: [DONE]\n\n"))
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestPDFLongQuestionIsAnswered(t *testing.T) {
+	r := newPDFRig(t)
+	m := fakeModel(t)
+	db := r.s.Digest.Cache
+	r.s.Digest = &digest.Engine{ChatURL: m.URL + "/chat", EmbedURL: m.URL + "/embed", Cache: db, Gate: digest.NewGate(2, 8, 5*time.Second), Client: m.Client()}
+	var pages []string
+	for p := 1; p <= 260; p++ { // ~260 pages × 40 lines: well over EmbedSyncMax chunks
+		pages = append(pages, pdftest.Lorem(p, 40))
+	}
+	o := newOrigin(t, "application/pdf", pdftest.Build(pdftest.Doc{Title: "Long", Pages: pages}))
+	code, out, _ := r.fetch(t, r.fleet, map[string]any{"url": o.srv.URL, "question": "what is on page three?", "context": false})
+	if code != 200 {
+		t.Fatalf("long question: %d %v, want 200 (P5a refused this)", code, out)
+	}
+	dg, _ := out["digest"].(map[string]any)
+	if dg == nil || dg["retrieval"] != "bm25" || dg["embedded"] != 0.0 {
+		t.Errorf("digest meta = %v, want retrieval bm25 and embedded 0 on the first question", dg)
+	}
+	if c, _ := out["content"].(string); !strings.Contains(c, "answer") {
+		t.Errorf("content %q", c)
+	}
+}
+
+func TestPDFSmallQuestionReportsRetrieval(t *testing.T) {
+	r := newPDFRig(t)
+	m := fakeModel(t)
+	db := r.s.Digest.Cache
+	r.s.Digest = &digest.Engine{ChatURL: m.URL + "/chat", EmbedURL: m.URL + "/embed", Cache: db, Gate: digest.NewGate(2, 8, 5*time.Second), Client: m.Client()}
+	o := newOrigin(t, "application/pdf", threePDF())
+	code, out, _ := r.fetch(t, r.fleet, map[string]any{"url": o.srv.URL, "question": "what is on page two?", "context": false})
+	dg, _ := out["digest"].(map[string]any)
+	if code != 200 || dg == nil || dg["retrieval"] != "whole" {
+		t.Errorf("small document: %d digest %v, want retrieval whole", code, dg)
+	}
+}

internal/api/pdf_test.go

@@ -396,7 +396,8 @@ func TestPDFQuestionTooLong(t *testing.T) {
 	if msg, _ := out["error"].(string); code != 422 || !strings.Contains(msg, "too long") || !strings.Contains(msg, "grep") {
 		t.Errorf("long question: %d %v; want 422 … too long … grep", code, out)
 	}
-	if DefaultPDFMaxQuestionChunks != 200 {
-		t.Errorf("DefaultPDFMaxQuestionChunks = %d, want 200", DefaultPDFMaxQuestionChunks)
+	// P5b: long documents are answered by retrieval now; the limit is only a safety cap.
+	if DefaultPDFMaxQuestionChunks != 5000 {
+		t.Errorf("DefaultPDFMaxQuestionChunks = %d, want 5000", DefaultPDFMaxQuestionChunks)
 	}
 }