diff --git a/cmd/sift/fetchcmd.go b/cmd/sift/fetchcmd.go index 8437591..47d56b0 100644 --- a/cmd/sift/fetchcmd.go +++ b/cmd/sift/fetchcmd.go @@ -5,6 +5,7 @@ import ( "encoding/json" "flag" "fmt" + "io" "net/http" "os" "path/filepath" @@ -54,42 +55,32 @@ func loadClient() (base, key string, err error) { } func fetchMain() { - fs := flag.NewFlagSet("fetch", flag.ExitOnError) - mode := fs.String("mode", "", "clean|raw|digest (default digest if --question)") - jq := fs.String("jq", "", "jq filter (JSON responses)") - question := fs.String("question", "", "focus the digest") - noCtx := fs.Bool("no-context", false, "skip Honcho context") - recoverFlag := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall") - noMain := fs.Bool("no-main", false, "do not isolate main content") - asJSON := fs.Bool("json", false, "print full /v1/fetch envelope") - _ = noMain - _ = fs.Parse(os.Args[1:]) - if fs.NArg() < 1 { - fmt.Fprintln(os.Stderr, "usage: sift fetch [--question Q] [--mode clean|raw|digest] [--recover] [--no-context] [--jq FILTER] [--json]") + rawURL, mode, jq, question, noCtx, recoverFlag, asJSON, err := parseFetchArgs(os.Args[1:]) + if err != nil { + fmt.Fprintln(os.Stderr, err) os.Exit(2) } - rawURL := fs.Arg(0) base, key, err := loadClient() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(5) } - modeVal := *mode + modeVal := mode if modeVal == "" { - if *question != "" { + if question != "" { modeVal = "digest" } else { modeVal = "clean" } } - wantCtx := *question != "" && !*noCtx + wantCtx := question != "" && !noCtx body, _ := json.Marshal(map[string]any{ "url": rawURL, "mode": modeVal, - "jq": *jq, - "question": *question, + "jq": jq, + "question": question, "context": wantCtx, - "recover": *recoverFlag, + "recover": recoverFlag, }) req, err := http.NewRequest(http.MethodPost, base+"/v1/fetch", bytes.NewReader(body)) if err != nil { @@ -119,7 +110,7 @@ func fetchMain() { } os.Exit(exitFor(resp.StatusCode, msg)) } - if *asJSON { + if asJSON { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") _ = enc.Encode(env) @@ -150,6 +141,63 @@ func fetchMain() { } } +const fetchUsage = "usage: sift fetch [--question Q] [--mode clean|raw|digest] [--recover] [--no-context] [--jq FILTER] [--json]" + +func parseFetchArgs(args []string) (rawURL, mode, jq, question string, noCtx, recoverFlag, asJSON bool, err error) { + flagArgs, pos, err := splitAroundPositional(args, fetchValueFlags) + if err != nil { + return "", "", "", "", false, false, false, err + } + fs := flag.NewFlagSet("fetch", flag.ContinueOnError) + fs.SetOutput(io.Discard) + modeF := fs.String("mode", "", "clean|raw|digest (default digest if --question)") + jqF := fs.String("jq", "", "jq filter (JSON responses)") + questionF := fs.String("question", "", "focus the digest") + noCtxF := fs.Bool("no-context", false, "skip Honcho context") + recoverF := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall") + _ = fs.Bool("no-main", false, "do not isolate main content") + asJSONF := fs.Bool("json", false, "print full /v1/fetch envelope") + if err := fs.Parse(flagArgs); err != nil { + return "", "", "", "", false, false, false, err + } + if fs.NArg() != 0 { + return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage) + } + if len(pos) != 1 { + return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage) + } + return pos[0], *modeF, *jqF, *questionF, *noCtxF, *recoverF, *asJSONF, nil +} + +var fetchValueFlags = map[string]bool{ + "mode": true, "jq": true, "question": true, +} + +func splitAroundPositional(args []string, valueFlags map[string]bool) (flagArgs, pos []string, err error) { + for i := 0; i < len(args); i++ { + a := args[i] + if a == "--" { + pos = append(pos, args[i+1:]...) + break + } + if strings.HasPrefix(a, "-") && a != "-" { + name := strings.TrimLeft(a, "-") + name, _, hasEq := strings.Cut(name, "=") + flagArgs = append(flagArgs, a) + if !hasEq && valueFlags[name] { + if i+1 >= len(args) { + return nil, nil, fmt.Errorf("flag needs an argument: -%s", name) + } + i++ + flagArgs = append(flagArgs, args[i]) + } + continue + } + pos = append(pos, a) + } + return flagArgs, pos, nil +} + func exitFor(status int, msg string) int { switch status { case 403: diff --git a/cmd/sift/fetchcmd_test.go b/cmd/sift/fetchcmd_test.go new file mode 100644 index 0000000..d3485e8 --- /dev/null +++ b/cmd/sift/fetchcmd_test.go @@ -0,0 +1,30 @@ +package main + +import "testing" + +func TestParseFetchFlagsAfterURL(t *testing.T) { + url, mode, _, q, noCtx, rec, _, err := parseFetchArgs([]string{"https://example.com/x", "--question", "what", "--mode", "digest", "--recover"}) + if err != nil { + t.Fatal(err) + } + if url != "https://example.com/x" || q != "what" || mode != "digest" || !rec || noCtx { + t.Fatalf("%q %q %q rec=%v", url, q, mode, rec) + } +} + +func TestParseFetchFlagsBeforeURL(t *testing.T) { + url, _, _, q, _, _, _, err := parseFetchArgs([]string{"--question", "why", "https://example.com/y"}) + if err != nil { + t.Fatal(err) + } + if url != "https://example.com/y" || q != "why" { + t.Fatalf("%q %q", url, q) + } +} + +func TestParseFetchRejectsExtraPositional(t *testing.T) { + _, _, _, _, _, _, _, err := parseFetchArgs([]string{"https://example.com/a", "https://example.com/b"}) + if err == nil { + t.Fatal("want error") + } +} diff --git a/internal/api/api.go b/internal/api/api.go index c5b5a98..ab3fcf1 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -1,6 +1,7 @@ package api import ( + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -24,7 +25,11 @@ import ( "git.wntrmute.dev/kyle/sift/internal/selfip" ) -const maxReqBody = 64 << 10 +const ( + maxReqBody = 64 << 10 + writeTimeout = 120 * time.Second + requestDeadline = 90 * time.Second +) type Server struct { Version string @@ -74,7 +79,7 @@ func (s *Server) HTTPServer(addr string) *http.Server { Handler: s, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, - WriteTimeout: 30 * time.Second, + WriteTimeout: writeTimeout, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 16 << 10, } @@ -82,6 +87,12 @@ func (s *Server) HTTPServer(addr string) *http.Server { func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxReqBody) + switch r.URL.Path { + case "/v1/fetch", "/v2/scrape": + ctx, cancel := context.WithTimeout(r.Context(), requestDeadline) + defer cancel() + r = r.WithContext(ctx) + } s.Mux.ServeHTTP(w, r) } diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 4a38434..abb0170 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strings" "testing" + "time" "git.wntrmute.dev/kyle/sift/internal/auth" "git.wntrmute.dev/kyle/sift/internal/fetch" @@ -240,3 +241,11 @@ func TestJQTooLarge(t *testing.T) { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } } + +func TestHTTPServerWriteTimeout(t *testing.T) { + s := New("test", &guard.Guard{}, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + hs := s.HTTPServer("127.0.0.1:0") + if hs.WriteTimeout != 120*time.Second { + t.Fatalf("WriteTimeout %s", hs.WriteTimeout) + } +}