kyleandornith-1.5-35b-a3b
8cdb3e8537
The digest now handles long documents itself (task 02), so the API's refusal limit is only a safety cap: DefaultPDFMaxQuestionChunks 200 -> 5000 (the 422 message text is unchanged). In v1fetch the digest-meta object gains "retrieval" and "embedded" next to chunks_used/chunks_total, only when the digest ran without a fallback. The PDF section explains a long-doc question is answered first from BM25 while its chunks embed in the background and hybrid on later questions; skill/SKILL.md gains one sentence. Given tests byte-identical. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
76 lines
2.8 KiB
Go
76 lines
2.8 KiB
Go
package api
|
||
|
||
// P5b task 03: a question on a long PDF is answered (by BM25 first) instead of refused, and the
|
||
// envelope says how the digest was built.
|
||
|
||
import (
|
||
"encoding/json"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"git.wntrmute.dev/kyle/sift/internal/digest"
|
||
"git.wntrmute.dev/kyle/sift/internal/pdftest"
|
||
)
|
||
|
||
// fakeModel answers /embed with a fixed vector and /chat with a one-word SSE reply.
|
||
func fakeModel(t *testing.T) *httptest.Server {
|
||
mux := http.NewServeMux()
|
||
mux.HandleFunc("/embed", func(w http.ResponseWriter, r *http.Request) {
|
||
var body struct {
|
||
Input []string `json:"input"`
|
||
}
|
||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||
var data []map[string]any
|
||
for i := range body.Input {
|
||
data = append(data, map[string]any{"index": i, "embedding": []float64{1, float64(i % 7)}})
|
||
}
|
||
_ = json.NewEncoder(w).Encode(map[string]any{"data": data})
|
||
})
|
||
mux.HandleFunc("/chat", func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/event-stream")
|
||
_, _ = w.Write([]byte(`data: {"choices":[{"delta":{"content":"answer (p. 3)"}}]}` + "\n\ndata: [DONE]\n\n"))
|
||
})
|
||
srv := httptest.NewServer(mux)
|
||
t.Cleanup(srv.Close)
|
||
return srv
|
||
}
|
||
|
||
func TestPDFLongQuestionIsAnswered(t *testing.T) {
|
||
r := newPDFRig(t)
|
||
m := fakeModel(t)
|
||
db := r.s.Digest.Cache
|
||
r.s.Digest = &digest.Engine{ChatURL: m.URL + "/chat", EmbedURL: m.URL + "/embed", Cache: db, Gate: digest.NewGate(2, 8, 5*time.Second), Client: m.Client()}
|
||
var pages []string
|
||
for p := 1; p <= 260; p++ { // ~260 pages × 40 lines: well over EmbedSyncMax chunks
|
||
pages = append(pages, pdftest.Lorem(p, 40))
|
||
}
|
||
o := newOrigin(t, "application/pdf", pdftest.Build(pdftest.Doc{Title: "Long", Pages: pages}))
|
||
code, out, _ := r.fetch(t, r.fleet, map[string]any{"url": o.srv.URL, "question": "what is on page three?", "context": false})
|
||
if code != 200 {
|
||
t.Fatalf("long question: %d %v, want 200 (P5a refused this)", code, out)
|
||
}
|
||
dg, _ := out["digest"].(map[string]any)
|
||
if dg == nil || dg["retrieval"] != "bm25" || dg["embedded"] != 0.0 {
|
||
t.Errorf("digest meta = %v, want retrieval bm25 and embedded 0 on the first question", dg)
|
||
}
|
||
if c, _ := out["content"].(string); !strings.Contains(c, "answer") {
|
||
t.Errorf("content %q", c)
|
||
}
|
||
}
|
||
|
||
func TestPDFSmallQuestionReportsRetrieval(t *testing.T) {
|
||
r := newPDFRig(t)
|
||
m := fakeModel(t)
|
||
db := r.s.Digest.Cache
|
||
r.s.Digest = &digest.Engine{ChatURL: m.URL + "/chat", EmbedURL: m.URL + "/embed", Cache: db, Gate: digest.NewGate(2, 8, 5*time.Second), Client: m.Client()}
|
||
o := newOrigin(t, "application/pdf", threePDF())
|
||
code, out, _ := r.fetch(t, r.fleet, map[string]any{"url": o.srv.URL, "question": "what is on page two?", "context": false})
|
||
dg, _ := out["digest"].(map[string]any)
|
||
if code != 200 || dg == nil || dg["retrieval"] != "whole" {
|
||
t.Errorf("small document: %d digest %v, want retrieval whole", code, dg)
|
||
}
|
||
}
|