kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

p1: cap jq output, log refusals, fail contract without SDK

Browse Source 
jq results are bounded (10k items / output cap) and return 400 when
exceeded. 401/403 log caller, host, and reason. make contract exits 1
unless firecrawl-py is present (SIFT_CONTRACT_SKIP_SDK=1 to skip).
This commit is contained in:
kyle
2026-09-22 22:02:38 -07:00
parent 9444a35322
commit 8e3a6cd339
6 changed files with 148 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/api.go
+25 -6
View File
@@ -119,11 +119,13 @@ func (s *Server) caller(w http.ResponseWriter, r *http.Request) *auth.Caller {
tok := strings.TrimSpace(strings.TrimPrefix(h, "Bearer"))
tok = strings.TrimSpace(tok)
if tok == "" || s.Auth == nil {
s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "missing key")
writeErr(w, 401, "unauthorized", "")
return nil
}
c := s.Auth.LookupBearer(tok)
if c == nil {
s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "unknown key")
writeErr(w, 401, "unauthorized", "")
return nil
}
@@ -164,7 +166,7 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
s.scrapeN.Add(1)
res, cl, ferr := s.doFetch(r, c, req.URL, "", onlyMain)
if ferr != nil {
s.writeFetchErr(w, ferr)
s.writeFetchErr(w, c, req.URL, ferr)
return
}
md, htmlOut := cl.Markdown, ""
@@ -225,7 +227,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
onlyMain := true
res, cl, ferr := s.doFetch(r, c, req.URL, req.JQ, onlyMain)
if ferr != nil {
s.writeFetchErr(w, ferr)
s.writeFetchErr(w, c, req.URL, ferr)
return
}
content := cl.Markdown
@@ -235,7 +237,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if mode == "raw" {
content = string(res.Body)
if len(content) > s.OutCap {
content = content[:s.OutCap]
content = clean.TruncateUTF8(content, s.OutCap)
cl.Trunc = true
cl.Warning = "truncated to output cap"
}
@@ -299,12 +301,21 @@ func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onl
cl, cerr = clean.Text(res, s.OutCap)
}
if cerr != nil {
var tl *clean.TooLargeError
if errors.As(cerr, &tl) {
return nil, nil, &fetch.Error{Status: 400, Msg: tl.Error()}
}
return nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
}
return res, cl, nil
}
func (s *Server) writeFetchErr(w http.ResponseWriter, err error) {
func (s *Server) writeFetchErr(w http.ResponseWriter, c *auth.Caller, rawURL string, err error) {
name := "-"
if c != nil {
name = c.Name
}
host := hostOf(rawURL)
var fe *fetch.Error
if errors.As(err, &fe) {
if fe.Status == 403 {
@@ -314,10 +325,15 @@ func (s *Server) writeFetchErr(w http.ResponseWriter, err error) {
fe.Status = 500
fe.Msg = "internal error"
}
if fe.Status == 401 || fe.Status == 403 {
s.Log.Warn("refused", "caller", name, "host", host, "reason", fe.Msg)
} else {
s.Log.Warn("fetch failed", "caller", name, "host", host, "reason", fe.Msg, "status", fe.Status)
}
writeErr(w, fe.Status, fe.Msg, fe.Hint)
return
}
s.Log.Error("internal", "err", err)
s.Log.Error("internal", "caller", name, "host", host, "err", err)
writeErr(w, 500, "internal error", "")
}
@@ -349,13 +365,16 @@ func (s *Server) release(caller, rawURL string) {
key := caller + "|" + hostOf(rawURL)
s.perMu.Lock()
ch := s.per[key]
s.perMu.Unlock()
if ch != nil {
select {
case <-ch:
default:
}
if len(ch) == 0 {
delete(s.per, key)
}
}
s.perMu.Unlock()
select {
case <-s.global:
default:
internal/api/api_test.go
+14
View File
@@ -217,3 +217,17 @@ func TestFetcherUsesSameGuard(t *testing.T) {
_ = key
_ = tls.VersionTLS13
}
func TestJQTooLarge(t *testing.T) {
g := &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`0`))
}))
defer ts.Close()
s, key := setupAPI(t, g)
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL, "jq": "range(20000)", "mode": "clean"})
if rec.Code != 400 || !strings.Contains(rec.Body.String(), "jq output too large") {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
}
internal/clean/clean.go
+32 -1
View File
@@ -9,6 +9,7 @@ import (
"net/url"
"strings"
"time"
"unicode/utf8"
readability "github.com/go-shiori/go-readability"
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
@@ -140,6 +141,7 @@ func JSON(res *fetch.Resource, jqFilter string, outCap int) (*Result, error) {
}
iter := code.RunWithContext(ctx, v)
var acc []any
var running int
for {
x, ok := iter.Next()
if !ok {
@@ -149,6 +151,17 @@ func JSON(res *fetch.Resource, jqFilter string, outCap int) (*Result, error) {
return nil, fmt.Errorf("jq: %w", err)
}
acc = append(acc, x)
if len(acc) > 10_000 {
return nil, &TooLargeError{Msg: "jq output too large"}
}
b, err := json.Marshal(x)
if err != nil {
return nil, fmt.Errorf("jq: %w", err)
}
running += len(b)
if running > outCap {
return nil, &TooLargeError{Msg: "jq output too large"}
}
}
switch len(acc) {
case 0:
@@ -183,15 +196,33 @@ func Text(res *fetch.Resource, outCap int) (*Result, error) {
return capResult(out, outCap), nil
}
// TooLargeError is a jq result that exceeded item or byte caps.
type TooLargeError struct{ Msg string }
func (e *TooLargeError) Error() string { return e.Msg }
func capResult(r *Result, capn int) *Result {
if len(r.Markdown) > capn {
r.Markdown = r.Markdown[:capn] + "\n…[truncated]"
r.Markdown = TruncateUTF8(r.Markdown, capn) + "\n…[truncated]"
r.Trunc = true
r.Warning = "truncated to output cap"
}
return r
}
func TruncateUTF8(s string, n int) string {
if n >= len(s) {
return s
}
if n < 0 {
return ""
}
for n > 0 && !utf8.RuneStart(s[n]) {
n--
}
return s[:n]
}
func stripDanger(s string) string {
doc, err := html.Parse(strings.NewReader(s))
if err != nil {
internal/fetch/fetch.go
+15 -8
View File
@@ -10,6 +10,7 @@ import (
"net/http"
"net/url"
"strings"
"sync"
"time"
"git.wntrmute.dev/kyle/sift/internal/guard"
@@ -60,6 +61,9 @@ type Fetcher struct {
UA string
MaxBody int64
Timeout time.Duration
clientOnce sync.Once
client *http.Client
}
func (f *Fetcher) maxBody() int64 {
@@ -83,7 +87,16 @@ func (f *Fetcher) ua() string {
return "sift/dev"
}
// Fetch GETs rawURL after Guard.CheckURL, using Guard.Client() from the same instance.
func (f *Fetcher) httpClient() *http.Client {
f.clientOnce.Do(func() {
c := f.Guard.Client()
c.Timeout = f.timeout()
f.client = c
})
return f.client
}
// Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard.
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
if f == nil || f.Guard == nil {
return nil, errf(500, "", "internal error")
@@ -103,10 +116,7 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
// no cookies, no auth, no Referer
client := f.Guard.Client()
client.Timeout = f.timeout()
resp, err := client.Do(req)
resp, err := f.httpClient().Do(req)
if err != nil {
return nil, mapDial(err)
}
@@ -126,9 +136,6 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
final = resp.Request.URL.String()
}
ct := resp.Header.Get("Content-Type")
if i := strings.IndexByte(ct, ';'); i >= 0 {
// keep full header for charset; also store raw
}
res := &Resource{
URL: rawURL,
FinalURL: final,