kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

p1: cap jq output, log refusals, fail contract without SDK

Browse Source 
jq results are bounded (10k items / output cap) and return 400 when
exceeded. 401/403 log caller, host, and reason. make contract exits 1
unless firecrawl-py is present (SIFT_CONTRACT_SKIP_SDK=1 to skip).
This commit is contained in:
kyle
2026-09-22 22:02:38 -07:00
parent 9444a35322
commit 8e3a6cd339
6 changed files with 148 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/api.go
+25 -6
View File
@@ -119,11 +119,13 @@ func (s *Server) caller(w http.ResponseWriter, r *http.Request) *auth.Caller {
tok := strings.TrimSpace(strings.TrimPrefix(h, "Bearer"))
tok = strings.TrimSpace(tok)
if tok == "" || s.Auth == nil {
s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "missing key")
writeErr(w, 401, "unauthorized", "")
return nil
}
c := s.Auth.LookupBearer(tok)
if c == nil {
s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "unknown key")
writeErr(w, 401, "unauthorized", "")
return nil
}
@@ -164,7 +166,7 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
s.scrapeN.Add(1)
res, cl, ferr := s.doFetch(r, c, req.URL, "", onlyMain)
if ferr != nil {
s.writeFetchErr(w, ferr)
s.writeFetchErr(w, c, req.URL, ferr)
return
}
md, htmlOut := cl.Markdown, ""
@@ -225,7 +227,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
onlyMain := true
res, cl, ferr := s.doFetch(r, c, req.URL, req.JQ, onlyMain)
if ferr != nil {
s.writeFetchErr(w, ferr)
s.writeFetchErr(w, c, req.URL, ferr)
return
}
content := cl.Markdown
@@ -235,7 +237,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
if mode == "raw" {
content = string(res.Body)
if len(content) > s.OutCap {
content = content[:s.OutCap]
content = clean.TruncateUTF8(content, s.OutCap)
cl.Trunc = true
cl.Warning = "truncated to output cap"
}
@@ -299,12 +301,21 @@ func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onl
cl, cerr = clean.Text(res, s.OutCap)
}
if cerr != nil {
var tl *clean.TooLargeError
if errors.As(cerr, &tl) {
return nil, nil, &fetch.Error{Status: 400, Msg: tl.Error()}
}
return nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
}
return res, cl, nil
}
func (s *Server) writeFetchErr(w http.ResponseWriter, err error) {
func (s *Server) writeFetchErr(w http.ResponseWriter, c *auth.Caller, rawURL string, err error) {
name := "-"
if c != nil {
name = c.Name
}
host := hostOf(rawURL)
var fe *fetch.Error
if errors.As(err, &fe) {
if fe.Status == 403 {
@@ -314,10 +325,15 @@ func (s *Server) writeFetchErr(w http.ResponseWriter, err error) {
fe.Status = 500
fe.Msg = "internal error"
}
if fe.Status == 401 || fe.Status == 403 {
s.Log.Warn("refused", "caller", name, "host", host, "reason", fe.Msg)
} else {
s.Log.Warn("fetch failed", "caller", name, "host", host, "reason", fe.Msg, "status", fe.Status)
}
writeErr(w, fe.Status, fe.Msg, fe.Hint)
return
}
s.Log.Error("internal", "err", err)
s.Log.Error("internal", "caller", name, "host", host, "err", err)
writeErr(w, 500, "internal error", "")
}
@@ -349,13 +365,16 @@ func (s *Server) release(caller, rawURL string) {
key := caller + "|" + hostOf(rawURL)
s.perMu.Lock()
ch := s.per[key]
s.perMu.Unlock()
if ch != nil {
select {
case <-ch:
default:
}
if len(ch) == 0 {
delete(s.per, key)
}
}
s.perMu.Unlock()
select {
case <-s.global:
default: