p1: cap jq output, log refusals, fail contract without SDK

jq results are bounded (10k items / output cap) and return 400 when
exceeded. 401/403 log caller, host, and reason. make contract exits 1
unless firecrawl-py is present (SIFT_CONTRACT_SKIP_SDK=1 to skip).
This commit is contained in:
2026-09-22 22:02:38 -07:00
parent 9444a35322
commit 8e3a6cd339
6 changed files with 148 additions and 48 deletions
+32 -1
View File
@@ -9,6 +9,7 @@ import (
"net/url"
"strings"
"time"
"unicode/utf8"
readability "github.com/go-shiori/go-readability"
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
@@ -140,6 +141,7 @@ func JSON(res *fetch.Resource, jqFilter string, outCap int) (*Result, error) {
}
iter := code.RunWithContext(ctx, v)
var acc []any
var running int
for {
x, ok := iter.Next()
if !ok {
@@ -149,6 +151,17 @@ func JSON(res *fetch.Resource, jqFilter string, outCap int) (*Result, error) {
return nil, fmt.Errorf("jq: %w", err)
}
acc = append(acc, x)
if len(acc) > 10_000 {
return nil, &TooLargeError{Msg: "jq output too large"}
}
b, err := json.Marshal(x)
if err != nil {
return nil, fmt.Errorf("jq: %w", err)
}
running += len(b)
if running > outCap {
return nil, &TooLargeError{Msg: "jq output too large"}
}
}
switch len(acc) {
case 0:
@@ -183,15 +196,33 @@ func Text(res *fetch.Resource, outCap int) (*Result, error) {
return capResult(out, outCap), nil
}
// TooLargeError is a jq result that exceeded item or byte caps.
type TooLargeError struct{ Msg string }
func (e *TooLargeError) Error() string { return e.Msg }
func capResult(r *Result, capn int) *Result {
if len(r.Markdown) > capn {
r.Markdown = r.Markdown[:capn] + "\n…[truncated]"
r.Markdown = TruncateUTF8(r.Markdown, capn) + "\n…[truncated]"
r.Trunc = true
r.Warning = "truncated to output cap"
}
return r
}
func TruncateUTF8(s string, n int) string {
if n >= len(s) {
return s
}
if n < 0 {
return ""
}
for n > 0 && !utf8.RuneStart(s[n]) {
n--
}
return s[:n]
}
func stripDanger(s string) string {
doc, err := html.Parse(strings.NewReader(s))
if err != nil {