p2: 120s write timeout and flags after the URL

WriteTimeout 30s was cutting digest mid-flight (queue+fetch+digest)
and tailscale then 502'd. 120s write, 90s handler deadline.
sift fetch URL --question now parses; extra positionals error.

cmd/sift/fetchcmd.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -54,42 +55,32 @@ func loadClient() (base, key string, err error) {
 }
 
 func fetchMain() {
-	fs := flag.NewFlagSet("fetch", flag.ExitOnError)
+	rawURL, mode, jq, question, noCtx, recoverFlag, asJSON, err := parseFetchArgs(os.Args[1:])
-	mode := fs.String("mode", "", "clean|raw|digest (default digest if --question)")
+	if err != nil {
-	jq := fs.String("jq", "", "jq filter (JSON responses)")
+		fmt.Fprintln(os.Stderr, err)
-	question := fs.String("question", "", "focus the digest")
-	noCtx := fs.Bool("no-context", false, "skip Honcho context")
-	recoverFlag := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall")
-	noMain := fs.Bool("no-main", false, "do not isolate main content")
-	asJSON := fs.Bool("json", false, "print full /v1/fetch envelope")
-	_ = noMain
-	_ = fs.Parse(os.Args[1:])
-	if fs.NArg() < 1 {
-		fmt.Fprintln(os.Stderr, "usage: sift fetch <url> [--question Q] [--mode clean|raw|digest] [--recover] [--no-context] [--jq FILTER] [--json]")
 		os.Exit(2)
 	}
-	rawURL := fs.Arg(0)
 	base, key, err := loadClient()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(5)
 	}
-	modeVal := *mode
+	modeVal := mode
 	if modeVal == "" {
-		if *question != "" {
+		if question != "" {
 			modeVal = "digest"
 		} else {
 			modeVal = "clean"
 		}
 	}
-	wantCtx := *question != "" && !*noCtx
+	wantCtx := question != "" && !noCtx
 	body, _ := json.Marshal(map[string]any{
 		"url":      rawURL,
 		"mode":     modeVal,
-		"jq":       *jq,
+		"jq":       jq,
-		"question": *question,
+		"question": question,
 		"context":  wantCtx,
-		"recover":  *recoverFlag,
+		"recover":  recoverFlag,
 	})
 	req, err := http.NewRequest(http.MethodPost, base+"/v1/fetch", bytes.NewReader(body))
 	if err != nil {
@@ -119,7 +110,7 @@ func fetchMain() {
 		}
 		os.Exit(exitFor(resp.StatusCode, msg))
 	}
-	if *asJSON {
+	if asJSON {
 		enc := json.NewEncoder(os.Stdout)
 		enc.SetIndent("", "  ")
 		_ = enc.Encode(env)
@@ -150,6 +141,63 @@ func fetchMain() {
 	}
 }
 
+const fetchUsage = "usage: sift fetch <url> [--question Q] [--mode clean|raw|digest] [--recover] [--no-context] [--jq FILTER] [--json]"
+
+func parseFetchArgs(args []string) (rawURL, mode, jq, question string, noCtx, recoverFlag, asJSON bool, err error) {
+	flagArgs, pos, err := splitAroundPositional(args, fetchValueFlags)
+	if err != nil {
+		return "", "", "", "", false, false, false, err
+	}
+	fs := flag.NewFlagSet("fetch", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	modeF := fs.String("mode", "", "clean|raw|digest (default digest if --question)")
+	jqF := fs.String("jq", "", "jq filter (JSON responses)")
+	questionF := fs.String("question", "", "focus the digest")
+	noCtxF := fs.Bool("no-context", false, "skip Honcho context")
+	recoverF := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall")
+	_ = fs.Bool("no-main", false, "do not isolate main content")
+	asJSONF := fs.Bool("json", false, "print full /v1/fetch envelope")
+	if err := fs.Parse(flagArgs); err != nil {
+		return "", "", "", "", false, false, false, err
+	}
+	if fs.NArg() != 0 {
+		return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage)
+	}
+	if len(pos) != 1 {
+		return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage)
+	}
+	return pos[0], *modeF, *jqF, *questionF, *noCtxF, *recoverF, *asJSONF, nil
+}
+
+var fetchValueFlags = map[string]bool{
+	"mode": true, "jq": true, "question": true,
+}
+
+func splitAroundPositional(args []string, valueFlags map[string]bool) (flagArgs, pos []string, err error) {
+	for i := 0; i < len(args); i++ {
+		a := args[i]
+		if a == "--" {
+			pos = append(pos, args[i+1:]...)
+			break
+		}
+		if strings.HasPrefix(a, "-") && a != "-" {
+			name := strings.TrimLeft(a, "-")
+			name, _, hasEq := strings.Cut(name, "=")
+			flagArgs = append(flagArgs, a)
+			if !hasEq && valueFlags[name] {
+				if i+1 >= len(args) {
+					return nil, nil, fmt.Errorf("flag needs an argument: -%s", name)
+				}
+				i++
+				flagArgs = append(flagArgs, args[i])
+			}
+			continue
+		}
+		pos = append(pos, a)
+	}
+	return flagArgs, pos, nil
+}
+
 func exitFor(status int, msg string) int {
 	switch status {
 	case 403:

cmd/sift/fetchcmd_test.go

@@ -0,0 +1,30 @@
+package main
+
+import "testing"
+
+func TestParseFetchFlagsAfterURL(t *testing.T) {
+	url, mode, _, q, noCtx, rec, _, err := parseFetchArgs([]string{"https://example.com/x", "--question", "what", "--mode", "digest", "--recover"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if url != "https://example.com/x" || q != "what" || mode != "digest" || !rec || noCtx {
+		t.Fatalf("%q %q %q rec=%v", url, q, mode, rec)
+	}
+}
+
+func TestParseFetchFlagsBeforeURL(t *testing.T) {
+	url, _, _, q, _, _, _, err := parseFetchArgs([]string{"--question", "why", "https://example.com/y"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if url != "https://example.com/y" || q != "why" {
+		t.Fatalf("%q %q", url, q)
+	}
+}
+
+func TestParseFetchRejectsExtraPositional(t *testing.T) {
+	_, _, _, _, _, _, _, err := parseFetchArgs([]string{"https://example.com/a", "https://example.com/b"})
+	if err == nil {
+		t.Fatal("want error")
+	}
+}

internal/api/api.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -24,7 +25,11 @@ import (
 	"git.wntrmute.dev/kyle/sift/internal/selfip"
 )
 
-const maxReqBody = 64 << 10
+const (
+	maxReqBody      = 64 << 10
+	writeTimeout    = 120 * time.Second
+	requestDeadline = 90 * time.Second
+)
 
 type Server struct {
 	Version string
@@ -74,7 +79,7 @@ func (s *Server) HTTPServer(addr string) *http.Server {
 		Handler:           s,
 		ReadHeaderTimeout: 5 * time.Second,
 		ReadTimeout:       15 * time.Second,
-		WriteTimeout:      30 * time.Second,
+		WriteTimeout:      writeTimeout,
 		IdleTimeout:       30 * time.Second,
 		MaxHeaderBytes:    16 << 10,
 	}
@@ -82,6 +87,12 @@ func (s *Server) HTTPServer(addr string) *http.Server {
 
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	r.Body = http.MaxBytesReader(w, r.Body, maxReqBody)
+	switch r.URL.Path {
+	case "/v1/fetch", "/v2/scrape":
+		ctx, cancel := context.WithTimeout(r.Context(), requestDeadline)
+		defer cancel()
+		r = r.WithContext(ctx)
+	}
 	s.Mux.ServeHTTP(w, r)
 }
 

internal/api/api_test.go

@@ -13,6 +13,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"git.wntrmute.dev/kyle/sift/internal/auth"
 	"git.wntrmute.dev/kyle/sift/internal/fetch"
@@ -240,3 +241,11 @@ func TestJQTooLarge(t *testing.T) {
 		t.Fatalf("%d %s", rec.Code, rec.Body.String())
 	}
 }
+
+func TestHTTPServerWriteTimeout(t *testing.T) {
+	s := New("test", &guard.Guard{}, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	hs := s.HTTPServer("127.0.0.1:0")
+	if hs.WriteTimeout != 120*time.Second {
+		t.Fatalf("WriteTimeout %s", hs.WriteTimeout)
+	}
+}