kyleandClaude Opus 5.5
218eaa8b53
Kyle: "Fold them into the plan." What changed and why is in docs/P6.md §14. In short:
- FetchFunc gets the *auth.Caller (the given api test could not compile, and a fetch with no
caller would bypass per-caller gates, the pool limit, the cache class and redaction);
- store reads never bump the lease (task 01's rule contradicted its own test); the API Touches;
- 1-2 workers round-robin over live jobs, so a batch holds at most 2 of the 8 global slots;
busy targets retry in the runner (bounded); batch PDFs use their own pool name; 210 s deadline;
- the api tests no longer touch the network or race the cancel; private names via fake DNS;
- failed/cancelled jobs are not reused; GET is a summary plus GET …/{idx} and ?results=1;
- the 256 KiB body cap moves into ServeHTTP; a fixed metrics label for /v1/batch/{id};
- guard.PreCheck (given verbatim) replaces guard.ParseURL at submit;
- 5 tasks became 8 (refactor, wiring, endpoints and serve/config split), each restating the
APIs it uses; every given test compiles against a panic-only skeleton and fails on it.
P6 starts after P5c: task 04 moves v1fetch, which P5b changes. Recheck the api given tests
against post-P5c master before the run.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
95 lines
3.2 KiB
Go
95 lines
3.2 KiB
Go
package api
|
|
|
|
// P6 task 04: v1fetch's per-target pipeline is extracted into s.fetchTarget, which the batch
|
|
// runner also calls. These tests pin that the two agree; every existing v1fetch test must also
|
|
// pass unchanged.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/netip"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.wntrmute.dev/kyle/sift/internal/auth"
|
|
"git.wntrmute.dev/kyle/sift/internal/batch"
|
|
"git.wntrmute.dev/kyle/sift/internal/fetch"
|
|
"git.wntrmute.dev/kyle/sift/internal/guard"
|
|
)
|
|
|
|
// newTargetServer is setupAPI with loopback allowed; its key belongs to caller "test".
|
|
func newTargetServer(t *testing.T) (*Server, string) {
|
|
t.Helper()
|
|
return setupAPI(t, &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}})
|
|
}
|
|
|
|
func TestFetchTargetMatchesV1Fetch(t *testing.T) {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<html><head><title>Same</title></head><body><article><h1>Same</h1><p>one pipeline, two callers</p></article></body></html>`))
|
|
}))
|
|
defer ts.Close()
|
|
s, key := newTargetServer(t)
|
|
c := &auth.Caller{Name: "test", Workspace: "hermes-fleet"}
|
|
|
|
res, err := s.fetchTarget(context.Background(), c, batch.Target{URL: ts.URL})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(res.Content, "one pipeline, two callers") {
|
|
t.Fatalf("content %q", res.Content)
|
|
}
|
|
if _, has := res.Meta["content"]; has {
|
|
t.Fatal("Meta must not repeat the content")
|
|
}
|
|
for _, k := range []string{"mode", "untrusted", "source", "warnings", "sizes", "timings"} {
|
|
if _, has := res.Meta[k]; !has {
|
|
t.Errorf("Meta lacks %q (it carries every envelope field except content)", k)
|
|
}
|
|
}
|
|
if res.Meta["mode"] != "clean" || res.Meta["untrusted"] != true {
|
|
t.Fatalf("meta %v", res.Meta)
|
|
}
|
|
|
|
// The same URL through /v1/fetch: identical content and mode, and the envelope has exactly
|
|
// the fields of Meta plus content.
|
|
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL})
|
|
if rec.Code != 200 {
|
|
t.Fatalf("v1fetch %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
var env map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if env["content"] != res.Content || env["mode"] != res.Meta["mode"] {
|
|
t.Fatalf("v1fetch and fetchTarget disagree:\n%v\n%+v", env, res)
|
|
}
|
|
if len(env) != len(res.Meta)+1 {
|
|
t.Fatalf("envelope keys %d, Meta keys %d: want Meta + content", len(env), len(res.Meta))
|
|
}
|
|
}
|
|
|
|
func TestFetchTargetErrors(t *testing.T) {
|
|
s, _ := newTargetServer(t)
|
|
c := &auth.Caller{Name: "test", Workspace: "hermes-fleet"}
|
|
cases := []struct {
|
|
name string
|
|
target batch.Target
|
|
status int
|
|
msg string
|
|
}{
|
|
{"blocked", batch.Target{URL: "http://10.0.0.5/"}, 403, "blocked"},
|
|
{"bad mode", batch.Target{URL: "http://127.0.0.1/", Options: batch.Options{Mode: "shout"}}, 400, ""},
|
|
}
|
|
for _, tc := range cases {
|
|
_, err := s.fetchTarget(context.Background(), c, tc.target)
|
|
var fe *fetch.Error
|
|
if !errors.As(err, &fe) || fe.Status != tc.status || !strings.Contains(fe.Msg, tc.msg) {
|
|
t.Errorf("%s: err %v, want a *fetch.Error %d containing %q", tc.name, err, tc.status, tc.msg)
|
|
}
|
|
}
|
|
}
|