p1: cap jq output, log refusals, fail contract without SDK
jq results are bounded (10k items / output cap) and return 400 when exceeded. 401/403 log caller, host, and reason. make contract exits 1 unless firecrawl-py is present (SIFT_CONTRACT_SKIP_SDK=1 to skip).
This commit is contained in:
+15
-8
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/guard"
|
||||
@@ -60,6 +61,9 @@ type Fetcher struct {
|
||||
UA string
|
||||
MaxBody int64
|
||||
Timeout time.Duration
|
||||
|
||||
clientOnce sync.Once
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (f *Fetcher) maxBody() int64 {
|
||||
@@ -83,7 +87,16 @@ func (f *Fetcher) ua() string {
|
||||
return "sift/dev"
|
||||
}
|
||||
|
||||
// Fetch GETs rawURL after Guard.CheckURL, using Guard.Client() from the same instance.
|
||||
func (f *Fetcher) httpClient() *http.Client {
|
||||
f.clientOnce.Do(func() {
|
||||
c := f.Guard.Client()
|
||||
c.Timeout = f.timeout()
|
||||
f.client = c
|
||||
})
|
||||
return f.client
|
||||
}
|
||||
|
||||
// Fetch GETs rawURL after Guard.CheckURL, using a Client built once from the same Guard.
|
||||
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
|
||||
if f == nil || f.Guard == nil {
|
||||
return nil, errf(500, "", "internal error")
|
||||
@@ -103,10 +116,7 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1")
|
||||
// no cookies, no auth, no Referer
|
||||
|
||||
client := f.Guard.Client()
|
||||
client.Timeout = f.timeout()
|
||||
|
||||
resp, err := client.Do(req)
|
||||
resp, err := f.httpClient().Do(req)
|
||||
if err != nil {
|
||||
return nil, mapDial(err)
|
||||
}
|
||||
@@ -126,9 +136,6 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Resource, error) {
|
||||
final = resp.Request.URL.String()
|
||||
}
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
if i := strings.IndexByte(ct, ';'); i >= 0 {
|
||||
// keep full header for charset; also store raw
|
||||
}
|
||||
res := &Resource{
|
||||
URL: rawURL,
|
||||
FinalURL: final,
|
||||
|
||||
Reference in New Issue
Block a user