sift/cmd/sift/fetchcmd.go

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/pelletier/go-toml/v2"
)

type clientCfg struct {
	URL     string `toml:"url"`
	KeyFile string `toml:"key_file"`
}

func clientConfigPath() string {
	if p := os.Getenv("SIFT_CLIENT"); p != "" {
		return p
	}
	if h := os.Getenv("HERMES_HOME"); h != "" {
		p := filepath.Join(h, "sift", "client.toml")
		if !hermesHomeIsDefault(h) {
			return p
		}
		if _, err := os.Stat(p); err == nil {
			return p
		}
	}
	home, err := os.UserHomeDir()
	if err != nil {
		return filepath.Join(".config", "sift", "client.toml")
	}
	return filepath.Join(home, ".config", "sift", "client.toml")
}

func hermesHomeIsDefault(h string) bool {
	home, err := os.UserHomeDir()
	if err != nil {
		return false
	}
	def, err := filepath.Abs(filepath.Join(home, ".hermes"))
	if err != nil {
		return false
	}
	got, err := filepath.Abs(h)
	if err != nil {
		return false
	}
	return got == def
}

func loadClient() (base, key string, err error) {
	home, err := os.UserHomeDir()
	if err != nil {
		return "", "", err
	}
	path := clientConfigPath()
	b, err := os.ReadFile(path)
	if err != nil {
		return "", "", fmt.Errorf("read %s: %w", path, err)
	}
	var c clientCfg
	if err := toml.Unmarshal(b, &c); err != nil {
		return "", "", err
	}
	if c.URL == "" || c.KeyFile == "" {
		return "", "", fmt.Errorf("client.toml: url and key_file required")
	}
	kf := c.KeyFile
	if strings.HasPrefix(kf, "~/") {
		kf = filepath.Join(home, kf[2:])
	}
	st, err := os.Stat(kf)
	if err != nil {
		return "", "", err
	}
	if st.Mode().Perm()&0o077 != 0 {
		return "", "", fmt.Errorf("key file %s is group/world readable", kf)
	}
	kb, err := os.ReadFile(kf)
	if err != nil {
		return "", "", err
	}
	return strings.TrimRight(c.URL, "/"), strings.TrimSpace(string(kb)), nil
}

func fetchMain() {
	o, err := parseFetchOpts(os.Args[1:])
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	base, key, err := loadClient()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(5)
	}
	body, _ := json.Marshal(fetchBody(o))
	req, err := http.NewRequest(http.MethodPost, base+"/v1/fetch", bytes.NewReader(body))
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(5)
	}
	req.Header.Set("Authorization", "Bearer "+key)
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(5)
	}
	defer resp.Body.Close()
	var env map[string]any
	if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(5)
	}
	if resp.StatusCode != 200 {
		msg, _ := env["error"].(string)
		hint, _ := env["hint"].(string)
		if hint != "" {
			fmt.Fprintf(os.Stderr, "%s\nhint: %s\n", msg, hint)
		} else {
			fmt.Fprintln(os.Stderr, msg)
		}
		os.Exit(exitFor(resp.StatusCode, msg))
	}
	if o.JSON {
		enc := json.NewEncoder(os.Stdout)
		enc.SetIndent("", "  ")
		_ = enc.Encode(env)
		return
	}
	if c, ok := env["content"].(string); ok {
		fmt.Print(c)
		if !strings.HasSuffix(c, "\n") {
			fmt.Println()
		}
	}
	if src, ok := env["source"].(map[string]any); ok {
		fmt.Println("--- sources ---")
		if u, ok := src["final_url"].(string); ok {
			fmt.Println(u)
		}
		if at, ok := src["snapshot_at"].(string); ok && at != "" {
			fmt.Println("snapshot_at:", at)
		}
	}
	if ctx, ok := env["context"].([]any); ok && len(ctx) > 0 {
		fmt.Println("--- prior context ---")
		fmt.Println("(prior knowledge, not page content)")
		for _, it := range ctx {
			m, _ := it.(map[string]any)
			fmt.Printf("- %v: %v\n", m["peer"], m["content"])
		}
	}
}

const fetchUsage = "usage: sift fetch <url> [--question Q] [--mode clean|raw|digest] [--pages a-b] [--grep RE] [--recover] [--no-context] [--jq FILTER] [--json]"

// fetchOpts is every flag `sift fetch` accepts plus the single positional URL.
type fetchOpts struct {
	URL, Mode, JQ, Question, Pages, Grep string
	NoContext, Recover, JSON             bool
}

// parseFetchOpts parses `sift fetch` args into fetchOpts. It knows which flags
// take a value; the URL is the one positional.
func parseFetchOpts(args []string) (fetchOpts, error) {
	flagArgs, pos, err := splitAroundPositional(args, fetchValueFlags)
	if err != nil {
		return fetchOpts{}, err
	}
	fs := flag.NewFlagSet("fetch", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	modeF := fs.String("mode", "", "clean|raw|digest (default digest if --question)")
	jqF := fs.String("jq", "", "jq filter (JSON responses)")
	questionF := fs.String("question", "", "focus the digest")
	pagesF := fs.String("pages", "", "only these pages of a PDF (a-b or a)")
	grepF := fs.String("grep", "", "only lines matching RE in a PDF")
	noCtxF := fs.Bool("no-context", false, "skip Honcho context")
	recoverF := fs.Bool("recover", false, "try Wayback / archive.today on bot-wall")
	_ = fs.Bool("no-main", false, "do not isolate main content")
	asJSONF := fs.Bool("json", false, "print full /v1/fetch envelope")
	if err := fs.Parse(flagArgs); err != nil {
		return fetchOpts{}, err
	}
	if fs.NArg() != 0 {
		return fetchOpts{}, fmt.Errorf("%s", fetchUsage)
	}
	if len(pos) != 1 {
		return fetchOpts{}, fmt.Errorf("%s", fetchUsage)
	}
	return fetchOpts{
		URL:       pos[0],
		Mode:      *modeF,
		JQ:        *jqF,
		Question:  *questionF,
		Pages:     *pagesF,
		Grep:      *grepF,
		NoContext: *noCtxF,
		Recover:   *recoverF,
		JSON:      *asJSONF,
	}, nil
}

// parseFetchArgs is the 8-value form the existing callers and tests use; it
// delegates to parseFetchOpts.
func parseFetchArgs(args []string) (rawURL, mode, jq, question string, noCtx, recoverFlag, asJSON bool, err error) {
	o, err := parseFetchOpts(args)
	if err != nil {
		return "", "", "", "", false, false, false, err
	}
	return o.URL, o.Mode, o.JQ, o.Question, o.NoContext, o.Recover, o.JSON, nil
}

// fetchBody builds the /v1/fetch request body. Mode defaults to digest when a
// question is set, else clean. Pages and grep are sent only when set.
func fetchBody(o fetchOpts) map[string]any {
	mode := o.Mode
	if mode == "" {
		if o.Question != "" {
			mode = "digest"
		} else {
			mode = "clean"
		}
	}
	body := map[string]any{
		"url":      o.URL,
		"mode":     mode,
		"jq":       o.JQ,
		"question": o.Question,
		"context":  o.Question != "" && !o.NoContext,
		"recover":  o.Recover,
	}
	if o.Pages != "" {
		body["pages"] = o.Pages
	}
	if o.Grep != "" {
		body["grep"] = o.Grep
	}
	return body
}

var fetchValueFlags = map[string]bool{
	"mode": true, "jq": true, "question": true, "pages": true, "grep": true,
}

func splitAroundPositional(args []string, valueFlags map[string]bool) (flagArgs, pos []string, err error) {
	for i := 0; i < len(args); i++ {
		a := args[i]
		if a == "--" {
			pos = append(pos, args[i+1:]...)
			break
		}
		if strings.HasPrefix(a, "-") && a != "-" {
			name := strings.TrimLeft(a, "-")
			name, _, hasEq := strings.Cut(name, "=")
			flagArgs = append(flagArgs, a)
			if !hasEq && valueFlags[name] {
				if i+1 >= len(args) {
					return nil, nil, fmt.Errorf("flag needs an argument: -%s", name)
				}
				i++
				flagArgs = append(flagArgs, args[i])
			}
			continue
		}
		pos = append(pos, a)
	}
	return flagArgs, pos, nil
}

func exitFor(status int, msg string) int {
	switch status {
	case 403:
		return 2
	case 415:
		return 4
	case 401, 500:
		return 5
	case 422, 413, 504, 429:
		return 3
	default:
		if strings.Contains(msg, "blocked:") {
			return 2
		}
		return 1
	}
}