kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dcache: PDF document cache (per URL and caller class, whole-document eviction)

Browse Source 
Add pdf_doc/pdf_page tables plus PDFEntry, PDFKey, PutPDF (one transaction,
old pages deleted first), GetPDF (page-ordered, updates last_hit) and TouchPDF
(304 revalidation moves only fetched_at). PDF documents count toward Size() and
are evicted whole by oldest use; PDFH/PDFM counters added. evict() now breaks
last_hit ties by rowid so same-second inserts evict in insertion order.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
kyle
2026-09-25 23:05:12 -07:00
co-authored by ornith-1.5-35b-a3b
parent f57aaef59e
commit b93b7d5816
4 changed files with 314 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/dcache/dcache.go
+29 -6
View File
@@ -45,6 +45,8 @@ type DB struct {
EmbedM uint64
SearchH uint64
SearchM uint64
PDFH uint64
PDFM uint64
Evictions uint64
}
@@ -92,6 +94,22 @@ func Open(path string, maxBytes int64, log *slog.Logger) (*DB, error) {
credits INTEGER NOT NULL,
PRIMARY KEY (source, day)
);
CREATE TABLE IF NOT EXISTS pdf_doc (
k TEXT PRIMARY KEY,
content_hash,
etag,
last_modified,
fetched_at INTEGER,
meta BLOB,
last_hit INTEGER,
nbytes INTEGER
);
CREATE TABLE IF NOT EXISTS pdf_page (
k TEXT,
page INTEGER,
text TEXT,
PRIMARY KEY (k, page)
);
`); err != nil {
_ = sq.Close()
return nil, err
@@ -230,11 +248,12 @@ func (d *DB) Size() int64 {
if d == nil {
return 0
}
var n, m, s sql.NullInt64
var n, m, s, p sql.NullInt64
_ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM digest`).Scan(&n)
_ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM embed`).Scan(&m)
_ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM search`).Scan(&s)
return n.Int64 + m.Int64 + s.Int64
_ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM pdf_doc`).Scan(&p)
return n.Int64 + m.Int64 + s.Int64 + p.Int64
}
func (d *DB) GetSearch(key string) (payload []byte, ok bool) {
@@ -313,18 +332,22 @@ func (d *DB) evict() {
var k string
err := d.sql.QueryRow(`
SELECT k FROM (
SELECT k, last_hit FROM digest
SELECT k, last_hit, rowid AS rid FROM digest
UNION ALL
SELECT k, last_hit FROM embed
SELECT k, last_hit, rowid AS rid FROM embed
UNION ALL
SELECT k, last_hit FROM search
) ORDER BY last_hit ASC LIMIT 1`).Scan(&k)
SELECT k, last_hit, rowid AS rid FROM search
UNION ALL
SELECT k, last_hit, rowid AS rid FROM pdf_doc
) ORDER BY last_hit ASC, rid ASC LIMIT 1`).Scan(&k)
if err != nil {
return
}
_, _ = d.sql.Exec(`DELETE FROM digest WHERE k = ?`, k)
_, _ = d.sql.Exec(`DELETE FROM embed WHERE k = ?`, k)
_, _ = d.sql.Exec(`DELETE FROM search WHERE k = ?`, k)
_, _ = d.sql.Exec(`DELETE FROM pdf_doc WHERE k = ?`, k)
_, _ = d.sql.Exec(`DELETE FROM pdf_page WHERE k = ?`, k)
d.mu.Lock()
d.Evictions++
d.mu.Unlock()