p1: cap jq output, log refusals, fail contract without SDK
jq results are bounded (10k items / output cap) and return 400 when exceeded. 401/403 log caller, host, and reason. make contract exits 1 unless firecrawl-py is present (SIFT_CONTRACT_SKIP_SDK=1 to skip).
This commit is contained in:
+32
-1
@@ -9,6 +9,7 @@ import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
readability "github.com/go-shiori/go-readability"
|
||||
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
@@ -140,6 +141,7 @@ func JSON(res *fetch.Resource, jqFilter string, outCap int) (*Result, error) {
|
||||
}
|
||||
iter := code.RunWithContext(ctx, v)
|
||||
var acc []any
|
||||
var running int
|
||||
for {
|
||||
x, ok := iter.Next()
|
||||
if !ok {
|
||||
@@ -149,6 +151,17 @@ func JSON(res *fetch.Resource, jqFilter string, outCap int) (*Result, error) {
|
||||
return nil, fmt.Errorf("jq: %w", err)
|
||||
}
|
||||
acc = append(acc, x)
|
||||
if len(acc) > 10_000 {
|
||||
return nil, &TooLargeError{Msg: "jq output too large"}
|
||||
}
|
||||
b, err := json.Marshal(x)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("jq: %w", err)
|
||||
}
|
||||
running += len(b)
|
||||
if running > outCap {
|
||||
return nil, &TooLargeError{Msg: "jq output too large"}
|
||||
}
|
||||
}
|
||||
switch len(acc) {
|
||||
case 0:
|
||||
@@ -183,15 +196,33 @@ func Text(res *fetch.Resource, outCap int) (*Result, error) {
|
||||
return capResult(out, outCap), nil
|
||||
}
|
||||
|
||||
// TooLargeError is a jq result that exceeded item or byte caps.
|
||||
type TooLargeError struct{ Msg string }
|
||||
|
||||
func (e *TooLargeError) Error() string { return e.Msg }
|
||||
|
||||
func capResult(r *Result, capn int) *Result {
|
||||
if len(r.Markdown) > capn {
|
||||
r.Markdown = r.Markdown[:capn] + "\n…[truncated]"
|
||||
r.Markdown = TruncateUTF8(r.Markdown, capn) + "\n…[truncated]"
|
||||
r.Trunc = true
|
||||
r.Warning = "truncated to output cap"
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func TruncateUTF8(s string, n int) string {
|
||||
if n >= len(s) {
|
||||
return s
|
||||
}
|
||||
if n < 0 {
|
||||
return ""
|
||||
}
|
||||
for n > 0 && !utf8.RuneStart(s[n]) {
|
||||
n--
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
func stripDanger(s string) string {
|
||||
doc, err := html.Parse(strings.NewReader(s))
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user