p2: 120s write timeout and flags after the URL
WriteTimeout 30s was cutting digest mid-flight (queue+fetch+digest) and tailscale then 502'd. 120s write, 90s handler deadline. sift fetch URL --question now parses; extra positionals error.
This commit is contained in:
+68
-20
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -54,42 +55,32 @@ func loadClient() (base, key string, err error) {
|
||||
}
|
||||
|
||||
func fetchMain() {
|
||||
fs := flag.NewFlagSet("fetch", flag.ExitOnError)
|
||||
mode := fs.String("mode", "", "clean|raw|digest (default digest if --question)")
|
||||
jq := fs.String("jq", "", "jq filter (JSON responses)")
|
||||
question := fs.String("question", "", "focus the digest")
|
||||
noCtx := fs.Bool("no-context", false, "skip Honcho context")
|
||||
recoverFlag := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall")
|
||||
noMain := fs.Bool("no-main", false, "do not isolate main content")
|
||||
asJSON := fs.Bool("json", false, "print full /v1/fetch envelope")
|
||||
_ = noMain
|
||||
_ = fs.Parse(os.Args[1:])
|
||||
if fs.NArg() < 1 {
|
||||
fmt.Fprintln(os.Stderr, "usage: sift fetch <url> [--question Q] [--mode clean|raw|digest] [--recover] [--no-context] [--jq FILTER] [--json]")
|
||||
rawURL, mode, jq, question, noCtx, recoverFlag, asJSON, err := parseFetchArgs(os.Args[1:])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
rawURL := fs.Arg(0)
|
||||
base, key, err := loadClient()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(5)
|
||||
}
|
||||
modeVal := *mode
|
||||
modeVal := mode
|
||||
if modeVal == "" {
|
||||
if *question != "" {
|
||||
if question != "" {
|
||||
modeVal = "digest"
|
||||
} else {
|
||||
modeVal = "clean"
|
||||
}
|
||||
}
|
||||
wantCtx := *question != "" && !*noCtx
|
||||
wantCtx := question != "" && !noCtx
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"url": rawURL,
|
||||
"mode": modeVal,
|
||||
"jq": *jq,
|
||||
"question": *question,
|
||||
"jq": jq,
|
||||
"question": question,
|
||||
"context": wantCtx,
|
||||
"recover": *recoverFlag,
|
||||
"recover": recoverFlag,
|
||||
})
|
||||
req, err := http.NewRequest(http.MethodPost, base+"/v1/fetch", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
@@ -119,7 +110,7 @@ func fetchMain() {
|
||||
}
|
||||
os.Exit(exitFor(resp.StatusCode, msg))
|
||||
}
|
||||
if *asJSON {
|
||||
if asJSON {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
_ = enc.Encode(env)
|
||||
@@ -150,6 +141,63 @@ func fetchMain() {
|
||||
}
|
||||
}
|
||||
|
||||
const fetchUsage = "usage: sift fetch <url> [--question Q] [--mode clean|raw|digest] [--recover] [--no-context] [--jq FILTER] [--json]"
|
||||
|
||||
func parseFetchArgs(args []string) (rawURL, mode, jq, question string, noCtx, recoverFlag, asJSON bool, err error) {
|
||||
flagArgs, pos, err := splitAroundPositional(args, fetchValueFlags)
|
||||
if err != nil {
|
||||
return "", "", "", "", false, false, false, err
|
||||
}
|
||||
fs := flag.NewFlagSet("fetch", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
modeF := fs.String("mode", "", "clean|raw|digest (default digest if --question)")
|
||||
jqF := fs.String("jq", "", "jq filter (JSON responses)")
|
||||
questionF := fs.String("question", "", "focus the digest")
|
||||
noCtxF := fs.Bool("no-context", false, "skip Honcho context")
|
||||
recoverF := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall")
|
||||
_ = fs.Bool("no-main", false, "do not isolate main content")
|
||||
asJSONF := fs.Bool("json", false, "print full /v1/fetch envelope")
|
||||
if err := fs.Parse(flagArgs); err != nil {
|
||||
return "", "", "", "", false, false, false, err
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage)
|
||||
}
|
||||
if len(pos) != 1 {
|
||||
return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage)
|
||||
}
|
||||
return pos[0], *modeF, *jqF, *questionF, *noCtxF, *recoverF, *asJSONF, nil
|
||||
}
|
||||
|
||||
var fetchValueFlags = map[string]bool{
|
||||
"mode": true, "jq": true, "question": true,
|
||||
}
|
||||
|
||||
func splitAroundPositional(args []string, valueFlags map[string]bool) (flagArgs, pos []string, err error) {
|
||||
for i := 0; i < len(args); i++ {
|
||||
a := args[i]
|
||||
if a == "--" {
|
||||
pos = append(pos, args[i+1:]...)
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(a, "-") && a != "-" {
|
||||
name := strings.TrimLeft(a, "-")
|
||||
name, _, hasEq := strings.Cut(name, "=")
|
||||
flagArgs = append(flagArgs, a)
|
||||
if !hasEq && valueFlags[name] {
|
||||
if i+1 >= len(args) {
|
||||
return nil, nil, fmt.Errorf("flag needs an argument: -%s", name)
|
||||
}
|
||||
i++
|
||||
flagArgs = append(flagArgs, args[i])
|
||||
}
|
||||
continue
|
||||
}
|
||||
pos = append(pos, a)
|
||||
}
|
||||
return flagArgs, pos, nil
|
||||
}
|
||||
|
||||
func exitFor(status int, msg string) int {
|
||||
switch status {
|
||||
case 403:
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseFetchFlagsAfterURL(t *testing.T) {
|
||||
url, mode, _, q, noCtx, rec, _, err := parseFetchArgs([]string{"https://example.com/x", "--question", "what", "--mode", "digest", "--recover"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if url != "https://example.com/x" || q != "what" || mode != "digest" || !rec || noCtx {
|
||||
t.Fatalf("%q %q %q rec=%v", url, q, mode, rec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFetchFlagsBeforeURL(t *testing.T) {
|
||||
url, _, _, q, _, _, _, err := parseFetchArgs([]string{"--question", "why", "https://example.com/y"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if url != "https://example.com/y" || q != "why" {
|
||||
t.Fatalf("%q %q", url, q)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFetchRejectsExtraPositional(t *testing.T) {
|
||||
_, _, _, _, _, _, _, err := parseFetchArgs([]string{"https://example.com/a", "https://example.com/b"})
|
||||
if err == nil {
|
||||
t.Fatal("want error")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user