kyleandornith-1.5-35b-a3b
0d923bc366
Move v1fetch's inline per-target pipeline into a new internal/api/target.go method, `fetchTarget(ctx, c, t batch.Target) (batch.Result, error)`, so the batch worker can drive the same work. The body moves unchanged in order: question truncation, mode default/check, wantCtx, doFetch, the recover path, the PDF branch (render, pages/grep errors, question-chunk limit), the clean/raw branch, digest, context search, the fetch log line and the envelope. `content` is returned as Result.Content; every other envelope key goes in Result.Meta, never content. v1fetch keeps reqN/caller/decode/400, builds a batch.Target from the request, calls fetchTarget, writeFetchErr on error and writeJSON(w, 200, env) on success where env is a copy of Meta plus "content". doFetch takes context.Context instead of *http.Request; scrape passes r.Context(). No new behaviour, fields or JSON keys. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
95 lines
3.2 KiB
Go
95 lines
3.2 KiB
Go
package api
|
|
|
|
// P6 task 05: v1fetch's per-target pipeline is extracted into s.fetchTarget, which the batch
|
|
// runner also calls. These tests pin that the two agree; every existing v1fetch test must also
|
|
// pass unchanged.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/netip"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.wntrmute.dev/kyle/sift/internal/auth"
|
|
"git.wntrmute.dev/kyle/sift/internal/batch"
|
|
"git.wntrmute.dev/kyle/sift/internal/fetch"
|
|
"git.wntrmute.dev/kyle/sift/internal/guard"
|
|
)
|
|
|
|
// newTargetServer is setupAPI with loopback allowed; its key belongs to caller "test".
|
|
func newTargetServer(t *testing.T) (*Server, string) {
|
|
t.Helper()
|
|
return setupAPI(t, &guard.Guard{AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}})
|
|
}
|
|
|
|
func TestFetchTargetMatchesV1Fetch(t *testing.T) {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<html><head><title>Same</title></head><body><article><h1>Same</h1><p>one pipeline, two callers</p></article></body></html>`))
|
|
}))
|
|
defer ts.Close()
|
|
s, key := newTargetServer(t)
|
|
c := &auth.Caller{Name: "test", Workspace: "hermes-fleet"}
|
|
|
|
res, err := s.fetchTarget(context.Background(), c, batch.Target{URL: ts.URL})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(res.Content, "one pipeline, two callers") {
|
|
t.Fatalf("content %q", res.Content)
|
|
}
|
|
if _, has := res.Meta["content"]; has {
|
|
t.Fatal("Meta must not repeat the content")
|
|
}
|
|
for _, k := range []string{"mode", "untrusted", "source", "warnings", "sizes", "timings"} {
|
|
if _, has := res.Meta[k]; !has {
|
|
t.Errorf("Meta lacks %q (it carries every envelope field except content)", k)
|
|
}
|
|
}
|
|
if res.Meta["mode"] != "clean" || res.Meta["untrusted"] != true {
|
|
t.Fatalf("meta %v", res.Meta)
|
|
}
|
|
|
|
// The same URL through /v1/fetch: identical content and mode, and the envelope has exactly
|
|
// the fields of Meta plus content.
|
|
rec := postJSON(t, s, "/v1/fetch", key, map[string]any{"url": ts.URL})
|
|
if rec.Code != 200 {
|
|
t.Fatalf("v1fetch %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
var env map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if env["content"] != res.Content || env["mode"] != res.Meta["mode"] {
|
|
t.Fatalf("v1fetch and fetchTarget disagree:\n%v\n%+v", env, res)
|
|
}
|
|
if len(env) != len(res.Meta)+1 {
|
|
t.Fatalf("envelope keys %d, Meta keys %d: want Meta + content", len(env), len(res.Meta))
|
|
}
|
|
}
|
|
|
|
func TestFetchTargetErrors(t *testing.T) {
|
|
s, _ := newTargetServer(t)
|
|
c := &auth.Caller{Name: "test", Workspace: "hermes-fleet"}
|
|
cases := []struct {
|
|
name string
|
|
target batch.Target
|
|
status int
|
|
msg string
|
|
}{
|
|
{"blocked", batch.Target{URL: "http://10.0.0.5/"}, 403, "blocked"},
|
|
{"bad mode", batch.Target{URL: "http://127.0.0.1/", Options: batch.Options{Mode: "shout"}}, 400, ""},
|
|
}
|
|
for _, tc := range cases {
|
|
_, err := s.fetchTarget(context.Background(), c, tc.target)
|
|
var fe *fetch.Error
|
|
if !errors.As(err, &fe) || fe.Status != tc.status || !strings.Contains(fe.Msg, tc.msg) {
|
|
t.Errorf("%s: err %v, want a *fetch.Error %d containing %q", tc.name, err, tc.status, tc.msg)
|
|
}
|
|
}
|
|
}
|