p1: cap jq output, log refusals, fail contract without SDK

jq results are bounded (10k items / output cap) and return 400 when
exceeded. 401/403 log caller, host, and reason. make contract exits 1
unless firecrawl-py is present (SIFT_CONTRACT_SKIP_SDK=1 to skip).
This commit is contained in:
2026-09-22 22:02:38 -07:00
parent 9444a35322
commit 8e3a6cd339
6 changed files with 148 additions and 48 deletions
+15 -8
View File
@@ -10,6 +10,7 @@ import (
"net/http"
"net/url"
"strings"
"sync"
"time"
"git.wntrmute.dev/kyle/sift/internal/guard"
@@ -60,6 +61,9 @@ type Fetcher struct {
UA string
MaxBody int64
Timeout time.Duration
clientOnce sync.Once
client *http.Client
}
func (f *Fetcher) maxBody() int64 {
@@ -83,7 +87,16 @@ func (f *Fetcher) ua() string {
return "sift/dev"
}
// Fetch GETs rawURL after Guard.CheckURL, using Guard.Client() from the same instance.
func (f *Fetcher) httpClient() *http.Client {
f.clientOnce.Do(func() {
c := f.Guard.Client()
c.Timeout = f.timeout()
f.client = c
})
return f.client
}
// Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard.
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
if f == nil || f.Guard == nil {
return nil, errf(500, "", "internal error")
@@ -103,10 +116,7 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
// no cookies, no auth, no Referer
client := f.Guard.Client()
client.Timeout = f.timeout()
resp, err := client.Do(req)
resp, err := f.httpClient().Do(req)
if err != nil {
return nil, mapDial(err)
}
@@ -126,9 +136,6 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
final = resp.Request.URL.String()
}
ct := resp.Header.Get("Content-Type")
if i := strings.IndexByte(ct, ';'); i >= 0 {
// keep full header for charset; also store raw
}
res := &Resource{
URL: rawURL,
FinalURL: final,