sift/cmd/sift/fetchcmd.go

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/pelletier/go-toml/v2"
)

type clientCfg struct {
	URL     string `toml:"url"`
	KeyFile string `toml:"key_file"`
}

func clientConfigPath() string {
	if p := os.Getenv("SIFT_CLIENT"); p != "" {
		return p
	}
	if h := os.Getenv("HERMES_HOME"); h != "" {
		p := filepath.Join(h, "sift", "client.toml")
		if _, err := os.Stat(p); err == nil {
			return p
		}
	}
	home, err := os.UserHomeDir()
	if err != nil {
		return filepath.Join(".config", "sift", "client.toml")
	}
	return filepath.Join(home, ".config", "sift", "client.toml")
}

func loadClient() (base, key string, err error) {
	home, err := os.UserHomeDir()
	if err != nil {
		return "", "", err
	}
	path := clientConfigPath()
	b, err := os.ReadFile(path)
	if err != nil {
		return "", "", fmt.Errorf("read %s: %w", path, err)
	}
	var c clientCfg
	if err := toml.Unmarshal(b, &c); err != nil {
		return "", "", err
	}
	if c.URL == "" || c.KeyFile == "" {
		return "", "", fmt.Errorf("client.toml: url and key_file required")
	}
	kf := c.KeyFile
	if strings.HasPrefix(kf, "~/") {
		kf = filepath.Join(home, kf[2:])
	}
	st, err := os.Stat(kf)
	if err != nil {
		return "", "", err
	}
	if st.Mode().Perm()&0o077 != 0 {
		return "", "", fmt.Errorf("key file %s is group/world readable", kf)
	}
	kb, err := os.ReadFile(kf)
	if err != nil {
		return "", "", err
	}
	return strings.TrimRight(c.URL, "/"), strings.TrimSpace(string(kb)), nil
}

func fetchMain() {
	rawURL, mode, jq, question, noCtx, recoverFlag, asJSON, err := parseFetchArgs(os.Args[1:])
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	base, key, err := loadClient()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(5)
	}
	modeVal := mode
	if modeVal == "" {
		if question != "" {
			modeVal = "digest"
		} else {
			modeVal = "clean"
		}
	}
	wantCtx := question != "" && !noCtx
	body, _ := json.Marshal(map[string]any{
		"url":      rawURL,
		"mode":     modeVal,
		"jq":       jq,
		"question": question,
		"context":  wantCtx,
		"recover":  recoverFlag,
	})
	req, err := http.NewRequest(http.MethodPost, base+"/v1/fetch", bytes.NewReader(body))
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(5)
	}
	req.Header.Set("Authorization", "Bearer "+key)
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(5)
	}
	defer resp.Body.Close()
	var env map[string]any
	if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(5)
	}
	if resp.StatusCode != 200 {
		msg, _ := env["error"].(string)
		hint, _ := env["hint"].(string)
		if hint != "" {
			fmt.Fprintf(os.Stderr, "%s\nhint: %s\n", msg, hint)
		} else {
			fmt.Fprintln(os.Stderr, msg)
		}
		os.Exit(exitFor(resp.StatusCode, msg))
	}
	if asJSON {
		enc := json.NewEncoder(os.Stdout)
		enc.SetIndent("", "  ")
		_ = enc.Encode(env)
		return
	}
	if c, ok := env["content"].(string); ok {
		fmt.Print(c)
		if !strings.HasSuffix(c, "\n") {
			fmt.Println()
		}
	}
	if src, ok := env["source"].(map[string]any); ok {
		fmt.Println("--- sources ---")
		if u, ok := src["final_url"].(string); ok {
			fmt.Println(u)
		}
		if at, ok := src["snapshot_at"].(string); ok && at != "" {
			fmt.Println("snapshot_at:", at)
		}
	}
	if ctx, ok := env["context"].([]any); ok && len(ctx) > 0 {
		fmt.Println("--- prior context ---")
		fmt.Println("(prior knowledge, not page content)")
		for _, it := range ctx {
			m, _ := it.(map[string]any)
			fmt.Printf("- %v: %v\n", m["peer"], m["content"])
		}
	}
}

const fetchUsage = "usage: sift fetch <url> [--question Q] [--mode clean|raw|digest] [--recover] [--no-context] [--jq FILTER] [--json]"

func parseFetchArgs(args []string) (rawURL, mode, jq, question string, noCtx, recoverFlag, asJSON bool, err error) {
	flagArgs, pos, err := splitAroundPositional(args, fetchValueFlags)
	if err != nil {
		return "", "", "", "", false, false, false, err
	}
	fs := flag.NewFlagSet("fetch", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	modeF := fs.String("mode", "", "clean|raw|digest (default digest if --question)")
	jqF := fs.String("jq", "", "jq filter (JSON responses)")
	questionF := fs.String("question", "", "focus the digest")
	noCtxF := fs.Bool("no-context", false, "skip Honcho context")
	recoverF := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall")
	_ = fs.Bool("no-main", false, "do not isolate main content")
	asJSONF := fs.Bool("json", false, "print full /v1/fetch envelope")
	if err := fs.Parse(flagArgs); err != nil {
		return "", "", "", "", false, false, false, err
	}
	if fs.NArg() != 0 {
		return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage)
	}
	if len(pos) != 1 {
		return "", "", "", "", false, false, false, fmt.Errorf("%s", fetchUsage)
	}
	return pos[0], *modeF, *jqF, *questionF, *noCtxF, *recoverF, *asJSONF, nil
}

var fetchValueFlags = map[string]bool{
	"mode": true, "jq": true, "question": true,
}

func splitAroundPositional(args []string, valueFlags map[string]bool) (flagArgs, pos []string, err error) {
	for i := 0; i < len(args); i++ {
		a := args[i]
		if a == "--" {
			pos = append(pos, args[i+1:]...)
			break
		}
		if strings.HasPrefix(a, "-") && a != "-" {
			name := strings.TrimLeft(a, "-")
			name, _, hasEq := strings.Cut(name, "=")
			flagArgs = append(flagArgs, a)
			if !hasEq && valueFlags[name] {
				if i+1 >= len(args) {
					return nil, nil, fmt.Errorf("flag needs an argument: -%s", name)
				}
				i++
				flagArgs = append(flagArgs, args[i])
			}
			continue
		}
		pos = append(pos, a)
	}
	return flagArgs, pos, nil
}

func exitFor(status int, msg string) int {
	switch status {
	case 403:
		return 2
	case 415:
		return 4
	case 401, 500:
		return 5
	case 422, 413, 504, 429:
		return 3
	default:
		if strings.Contains(msg, "blocked:") {
			return 2
		}
		return 1
	}
}