package api

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"log/slog"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"git.wntrmute.dev/kyle/sift/internal/auth"
	"git.wntrmute.dev/kyle/sift/internal/batch"
	"git.wntrmute.dev/kyle/sift/internal/clean"
	"git.wntrmute.dev/kyle/sift/internal/dcache"
	"git.wntrmute.dev/kyle/sift/internal/digest"
	"git.wntrmute.dev/kyle/sift/internal/fetch"
	"git.wntrmute.dev/kyle/sift/internal/fleetctx"
	"git.wntrmute.dev/kyle/sift/internal/guard"
	"git.wntrmute.dev/kyle/sift/internal/metrics"
	"git.wntrmute.dev/kyle/sift/internal/pdfdoc"
	"git.wntrmute.dev/kyle/sift/internal/pdfpool"
	"git.wntrmute.dev/kyle/sift/internal/pdftext"
	"git.wntrmute.dev/kyle/sift/internal/search"
	"git.wntrmute.dev/kyle/sift/internal/selfip"
)

const (
	maxReqBody      = 64 << 10
	writeTimeout    = 240 * time.Second // P5a: a PDF may take 120 s to download + 20 s queued + 60 s to extract
	requestDeadline = 210 * time.Second // P5a: the PDF path's worst case (see writeTimeout); other paths keep their own shorter timeouts

	// DefaultPDFMaxUpload is the largest body POST /v1/extract accepts: an uploaded
	// PDF, addressed by its hash and extracted once per content.
	DefaultPDFMaxUpload = 100 << 20
)

type Server struct {
	Version string
	Log     *slog.Logger
	Guard   *guard.Guard
	Fetch   *fetch.Fetcher
	Auth    *auth.Store
	Mux     *http.ServeMux
	OutCap  int
	SelfIP  *selfip.Discoverer
	Digest  *digest.Engine

	PDFRunner            *pdftext.Runner
	PDFPool              *pdfpool.Pool
	PDFTTL               time.Duration
	PDFMaxQuestionChunks int
	PDFMaxUpload         int64 // 0 = DefaultPDFMaxUpload; the /v1/extract upload cap
	Search               *search.Engine
	Ctx                  *fleetctx.Client
	// PermitRestricted is context.permit_restricted: those callers never
	// have a fetch host written to the log (host=-). Independent of Honcho.
	PermitRestricted []string
	Prom             *metrics.Set

	global chan struct{}
	perMu  sync.Mutex
	per    map[string]chan struct{}

	reqN, blockN, scrapeN atomic.Uint64

	pdfPoolOnce sync.Once

	// uploads bounds how many /v1/extract bodies (up to PDFMaxUpload each) are read into
	// memory at once, ahead of the PDF pool's own limits.
	uploads chan struct{}
}

func New(version string, g *guard.Guard, f *fetch.Fetcher, keys *auth.Store, log *slog.Logger) *Server {
	if log == nil {
		log = slog.Default()
	}
	s := &Server{
		Version: version,
		Log:     log,
		Guard:   g,
		Fetch:   f,
		Auth:    keys,
		Mux:     http.NewServeMux(),
		OutCap:  200_000,
		global:  make(chan struct{}, 8),
		per:     map[string]chan struct{}{},
		uploads: make(chan struct{}, maxUploads),
	}
	s.Prom = metrics.New(s)
	s.Mux.HandleFunc("GET /healthz", s.healthz)
	s.Mux.HandleFunc("GET /healthz/", s.healthz)
	s.Mux.HandleFunc("GET /metrics", s.metrics)
	s.Mux.HandleFunc("GET /metrics.json", s.metrics)
	s.Mux.HandleFunc("POST /v2/scrape", s.scrape)
	s.Mux.HandleFunc("POST /v1/fetch", s.v1fetch)
	s.Mux.HandleFunc("POST /v1/extract", s.v1extract)
	s.Mux.HandleFunc("POST /v1/search", s.v1search)
	s.Mux.HandleFunc("POST /v2/search", s.v2search)
	return s
}

func (s *Server) HTTPServer(addr string) *http.Server {
	return &http.Server{
		Addr:              addr,
		Handler:           s,
		ReadHeaderTimeout: 5 * time.Second,
		ReadTimeout:       15 * time.Second,
		WriteTimeout:      writeTimeout,
		IdleTimeout:       30 * time.Second,
		MaxHeaderBytes:    16 << 10,
	}
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var limit int64 = maxReqBody
	if r.URL.Path == "/v1/extract" {
		limit = int64(s.pdfMaxUpload())
	}
	r.Body = http.MaxBytesReader(w, r.Body, limit)
	switch r.URL.Path {
	case "/v1/fetch", "/v2/scrape", "/v1/search", "/v2/search", "/v1/extract":
		ctx, cancel := context.WithTimeout(r.Context(), requestDeadline)
		defer cancel()
		r = r.WithContext(ctx)
		cw := &codeWriter{ResponseWriter: w}
		start := time.Now()
		s.Mux.ServeHTTP(cw, r)
		code := cw.code
		if code == 0 {
			code = http.StatusOK
		}
		s.Prom.Observe(r.URL.Path, time.Since(start), code)
		return
	}
	s.Mux.ServeHTTP(w, r)
}

type codeWriter struct {
	http.ResponseWriter
	code int
}

func (c *codeWriter) WriteHeader(status int) {
	if c.code == 0 {
		c.code = status
	}
	c.ResponseWriter.WriteHeader(status)
}

func (c *codeWriter) Write(b []byte) (int, error) {
	if c.code == 0 {
		c.code = http.StatusOK
	}
	return c.ResponseWriter.Write(b)
}

func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	out := map[string]any{"status": "ok", "version": s.Version}
	if s.SelfIP != nil {
		out["self_ip"] = s.SelfIP.Status()
	}
	if s.Search != nil {
		if d := s.Search.Health(); len(d) > 0 {
			out["status"] = "degraded"
			out["search"] = d
		}
	}
	_ = json.NewEncoder(w).Encode(out)
}

func (s *Server) metrics(w http.ResponseWriter, r *http.Request) {
	if !s.allowMetrics(w, r) {
		return
	}
	if r.URL.Path == "/metrics.json" || r.URL.Query().Get("format") == "json" {
		s.writeMetricsJSON(w)
		return
	}
	s.Prom.Handler().ServeHTTP(w, r)
}

func (s *Server) allowMetrics(w http.ResponseWriter, r *http.Request) bool {
	if metrics.LoopbackOpen(r) {
		return true
	}
	h := r.Header.Get("Authorization")
	tok := strings.TrimSpace(strings.TrimPrefix(h, "Bearer"))
	tok = strings.TrimSpace(tok)
	if tok == "" || s.Auth == nil {
		s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "missing metrics key")
		writeErr(w, 401, "unauthorized", "")
		return false
	}
	c := s.Auth.LookupBearer(tok)
	if c == nil {
		s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "unknown metrics key")
		writeErr(w, 401, "unauthorized", "")
		return false
	}
	if !c.MetricsOnly() {
		s.Log.Warn("unauthorized", "caller", c.Name, "host", "-", "reason", "not metrics caller")
		writeErr(w, 403, "unauthorized", "")
		return false
	}
	return true
}

func (s *Server) writeMetricsJSON(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json")
	fails := uint64(0)
	if s.SelfIP != nil {
		fails = s.SelfIP.Failures.Load()
	}
	m := map[string]any{
		"requests":         s.reqN.Load(),
		"blocks":           s.blockN.Load(),
		"scrapes":          s.scrapeN.Load(),
		"self_ip_failures": fails,
	}
	if s.Search != nil {
		for k, v := range s.Search.Metrics() {
			m[k] = v
		}
	}
	calls, fb := uint64(0), map[string]uint64{}
	var inflight, waiters int
	if s.Digest != nil {
		calls, fb = s.Digest.Stats()
		inflight, waiters = s.Digest.Gate.Snapshot()
	}
	m["digest_calls"] = calls
	m["digest_fallbacks"] = fb
	m["dixie_inflight"] = inflight
	m["dixie_queue_waiters"] = waiters
	var dH, dM, eH, eM uint64
	if cache := s.cache(); cache != nil {
		dH, dM, eH, eM, _, _, _ = cache.Stats()
	}
	m["dcache_digest_hit"] = dH
	m["dcache_digest_miss"] = dM
	m["dcache_embed_hit"] = eH
	m["dcache_embed_miss"] = eM
	_ = json.NewEncoder(w).Encode(m)
}

func (s *Server) cache() *dcache.DB {
	if s.Digest != nil && s.Digest.Cache != nil {
		return s.Digest.Cache
	}
	if s.Search != nil {
		return s.Search.Cache
	}
	return nil
}

func (s *Server) Snapshot() metrics.Snapshot {
	out := metrics.Snapshot{
		Blocks:     s.blockN.Load(),
		Scrapes:    s.scrapeN.Load(),
		Restricted: map[string]struct{}{},
	}
	for _, n := range s.PermitRestricted {
		out.Restricted[n] = struct{}{}
	}
	if s.SelfIP != nil {
		out.SelfIPFailures = s.SelfIP.Failures.Load()
	}
	if s.Digest != nil {
		out.DigestCalls, out.DigestFallbacks = s.Digest.Stats()
		out.DixieInflight, out.DixieWaiters = s.Digest.Gate.Snapshot()
	}
	if cache := s.cache(); cache != nil {
		out.DigestHit, out.DigestMiss, out.EmbedHit, out.EmbedMiss, _, _, _ = cache.Stats()
	}
	if s.Search != nil {
		m := s.Search.Metrics()
		if v, ok := m["search_queries"].(map[string]map[string]uint64); ok {
			out.SearchQueries = v
		}
		if v, ok := m["search_credits"].(map[string]map[string]uint64); ok {
			out.SearchCredits = v
		}
		if v, ok := m["search_daily_credits"].(map[string]uint64); ok {
			out.SearchDaily = v
		}
		if v, ok := m["search_capped"].(map[string]uint64); ok {
			out.SearchCapped = v
		}
		if v, ok := m["search_cache_hit"].(uint64); ok {
			out.SearchHit = v
		}
		if v, ok := m["search_cache_miss"].(uint64); ok {
			out.SearchMiss = v
		}
	}
	return out
}

type scrapeReq struct {
	URL                 string   `json:"url"`
	Formats             []string `json:"formats"`
	OnlyMainContent     *bool    `json:"onlyMainContent"`
	SkipTlsVerification *bool    `json:"skipTlsVerification"`
	RemoveBase64Images  *bool    `json:"removeBase64Images"`
	FastMode            *bool    `json:"fastMode"`
	BlockAds            *bool    `json:"blockAds"`
	StoreInCache        *bool    `json:"storeInCache"`
	MaxAge              *int     `json:"maxAge"`
	Mobile              *bool    `json:"mobile"`
	Origin              string   `json:"origin"`
}

type v1Req struct {
	URL      string `json:"url"`
	Question string `json:"question"`
	Mode     string `json:"mode"`
	Context  *bool  `json:"context"`
	JQ       string `json:"jq"`
	Recover  bool   `json:"recover"`

	// Pages ("41-60") and Grep (an RE2 pattern) narrow a PDF answer in clean mode; see pdfdoc.
	Pages string `json:"pages"`
	Grep  string `json:"grep"`
}

func (s *Server) caller(w http.ResponseWriter, r *http.Request) *auth.Caller {
	h := r.Header.Get("Authorization")
	tok := strings.TrimSpace(strings.TrimPrefix(h, "Bearer"))
	tok = strings.TrimSpace(tok)
	if tok == "" || s.Auth == nil {
		s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "missing key")
		writeErr(w, 401, "unauthorized", "")
		return nil
	}
	c := s.Auth.LookupBearer(tok)
	if c == nil {
		s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "unknown key")
		writeErr(w, 401, "unauthorized", "")
		return nil
	}
	if c.MetricsOnly() {
		s.Log.Warn("unauthorized", "caller", c.Name, "host", "-", "reason", "metrics caller")
		writeErr(w, 403, "unauthorized", "")
		return nil
	}
	return c
}

func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
	s.reqN.Add(1)
	c := s.caller(w, r)
	if c == nil {
		return
	}
	var req scrapeReq
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.URL == "" {
		writeErr(w, 400, "invalid url", "")
		return
	}
	_ = req.SkipTlsVerification // MUST ignore; never disable TLS verify
	onlyMain := true
	if req.OnlyMainContent != nil {
		onlyMain = *req.OnlyMainContent
	}
	wantMD, wantHTML := true, false
	if len(req.Formats) > 0 {
		wantMD, wantHTML = false, false
		for _, f := range req.Formats {
			switch strings.ToLower(f) {
			case "markdown":
				wantMD = true
			case "html":
				wantHTML = true
			}
		}
		if !wantMD && !wantHTML {
			wantMD = true
		}
	}
	s.scrapeN.Add(1)
	res, cl, doc, ferr := s.doFetch(r.Context(), c, req.URL, "", onlyMain)
	if ferr != nil {
		s.writeFetchErr(w, c, req.URL, ferr)
		return
	}
	md, htmlOut := cl.Markdown, ""
	pages := 0
	if doc != nil {
		rp, err := s.renderPDF(doc, pdfdoc.Options{OutCap: s.OutCap})
		if err != nil {
			s.writeFetchErr(w, c, req.URL, err)
			return
		}
		md = rp.Content
		htmlOut = ""
		pages = doc.Pages
	}
	if wantHTML {
		htmlOut = cl.HTML
	}
	if !wantMD {
		md = ""
	}
	rawTok := len(res.Body) / 4
	retTok := len(md) / 4
	s.Log.Info("scrape", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "status", res.Status, "bytes", len(res.Body), "pdf_pages", pages)
	writeJSON(w, 200, map[string]any{
		"success": true,
		"data": map[string]any{
			"markdown": md,
			"html":     htmlOut,
			"metadata": map[string]any{
				"title":       cl.Title,
				"sourceURL":   res.FinalURL,
				"statusCode":  res.Status,
				"contentType": res.ContentType,
				"numPages":    pages,
				"sift": map[string]any{
					"hash":      res.SHA256(),
					"fetchedAt": res.FetchedAt.Format(time.RFC3339),
					"cached":    false,
					"truncated": cl.Trunc,
					"tokens":    map[string]int{"raw": rawTok, "returned": retTok},
				},
			},
		},
	})
}

func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
	s.reqN.Add(1)
	c := s.caller(w, r)
	if c == nil {
		return
	}
	var req v1Req
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.URL == "" {
		writeErr(w, 400, "invalid url", "")
		return
	}
	t := batch.Target{
		URL: req.URL,
		Options: batch.Options{
			Mode:     req.Mode,
			Question: req.Question,
			Pages:    req.Pages,
			Grep:     req.Grep,
			JQ:       req.JQ,
			Context:  req.Context,
			Recover:  req.Recover,
		},
	}
	res, err := s.fetchTarget(r.Context(), c, t)
	if err != nil {
		s.writeFetchErr(w, c, req.URL, err)
		return
	}
	env := make(map[string]any, len(res.Meta)+1)
	for k, v := range res.Meta {
		env[k] = v
	}
	env["content"] = res.Content
	writeJSON(w, 200, env)
}

func recoverWorthy(err error) bool {
	var fe *fetch.Error
	if !errors.As(err, &fe) {
		return false
	}
	if strings.Contains(fe.Msg, "blocked:") {
		return false
	}
	return fe.Hint == "browser" || fe.Status == 422 || fe.Status == 429 || fe.Status == 504
}

func headings(md string) []string {
	var h []string
	for _, line := range strings.Split(md, "\n") {
		if strings.HasPrefix(line, "#") {
			h = append(h, strings.TrimLeft(line, "# "))
		}
	}
	return h
}

func (s *Server) cleanFetched(res *fetch.Resource, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, *pdftext.Doc, error) {
	kind := clean.Kind(res.ContentType, res.Body)
	if kind == "other" || kind == "pdf" {
		// The recover path (archive snapshots) never serves a PDF.
		return nil, nil, nil, &fetch.Error{Status: 415, Hint: "browser", Msg: "unsupported content type"}
	}
	var cl *clean.Result
	var cerr error
	switch kind {
	case "html":
		cl, cerr = clean.HTML(res, onlyMain, s.OutCap)
	case "json":
		cl, cerr = clean.JSON(res, jq, s.OutCap)
	default:
		cl, cerr = clean.Text(res, s.OutCap)
	}
	if cerr != nil {
		return nil, nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
	}
	return res, cl, nil, nil
}

func (s *Server) doFetch(ctx context.Context, c *auth.Caller, rawURL, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, *pdftext.Doc, error) {
	res, hitDoc, ok, err := s.pdfFromCache(ctx, c, rawURL)
	if err != nil {
		return nil, nil, nil, err
	}
	if ok {
		return res, &clean.Result{Kind: "pdf", Title: hitDoc.Title}, hitDoc, nil
	}
	if res == nil {
		if err := s.acquire(c.Name, rawURL); err != nil {
			return nil, nil, nil, err
		}
		defer s.release(c.Name, rawURL)

		res, err = s.Fetch.Fetch(ctx, rawURL)
		if err != nil {
			return nil, nil, nil, err
		}
	}
	kind := clean.Kind(res.ContentType, res.Body)
	if kind == "other" {
		ct := res.ContentType
		if ct == "" {
			ct = "unknown"
		}
		return nil, nil, nil, &fetch.Error{Status: 415, Hint: "browser", Msg: "unsupported content type " + ct + "; hint: browser"}
	}
	var cl *clean.Result
	var cerr error
	switch kind {
	case "html":
		cl, cerr = clean.HTML(res, onlyMain, s.OutCap)
	case "json":
		cl, cerr = clean.JSON(res, jq, s.OutCap)
	case "text":
		cl, cerr = clean.Text(res, s.OutCap)
	}
	if cerr != nil {
		var tl *clean.TooLargeError
		if errors.As(cerr, &tl) {
			return nil, nil, nil, &fetch.Error{Status: 400, Msg: tl.Error()}
		}
		return nil, nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
	}
	var doc *pdftext.Doc
	if kind == "pdf" {
		doc, err = s.extractPDF(ctx, c, res)
		if err != nil {
			return nil, nil, nil, err
		}
		s.storePDF(dcache.PDFKey(rawURL, s.callerClass(c)), res, doc)
		cl = &clean.Result{Kind: "pdf", Title: doc.Title}
	}
	return res, cl, doc, nil
}

func (s *Server) writeFetchErr(w http.ResponseWriter, c *auth.Caller, rawURL string, err error) {
	name := "-"
	if c != nil {
		name = c.Name
	}
	host := s.logHost(c, rawURL)
	var fe *fetch.Error
	if errors.As(err, &fe) {
		if fe.Status == 403 {
			s.blockN.Add(1)
		}
		if fe.Status == 502 {
			fe.Status = 500
			fe.Msg = "internal error"
		}
		if fe.Status == 401 || fe.Status == 403 {
			s.Log.Warn("refused", "caller", name, "host", host, "reason", s.logReason(c, fe))
		} else {
			s.Log.Warn("fetch failed", "caller", name, "host", host, "reason", s.logReason(c, fe), "status", fe.Status)
		}
		if fe.RetryAfter > 0 {
			w.Header().Set("Retry-After", strconv.Itoa(fe.RetryAfter))
		}
		writeErr(w, fe.Status, fe.Msg, fe.Hint)
		return
	}
	s.Log.Error("internal", "caller", name, "host", host, "err", s.logErr(c, err))
	writeErr(w, 500, "internal error", "")
}

func (s *Server) acquire(caller, rawURL string) error {
	host := hostOf(rawURL)
	key := caller + "|" + host
	select {
	case s.global <- struct{}{}:
	case <-time.After(5 * time.Second):
		return &fetch.Error{Status: 429, Msg: "busy"}
	}
	s.perMu.Lock()
	ch, ok := s.per[key]
	if !ok {
		ch = make(chan struct{}, 2)
		s.per[key] = ch
	}
	s.perMu.Unlock()
	select {
	case ch <- struct{}{}:
		return nil
	case <-time.After(5 * time.Second):
		<-s.global
		return &fetch.Error{Status: 429, Msg: "busy"}
	}
}

func (s *Server) release(caller, rawURL string) {
	key := caller + "|" + hostOf(rawURL)
	s.perMu.Lock()
	ch := s.per[key]
	if ch != nil {
		select {
		case <-ch:
		default:
		}
		if len(ch) == 0 {
			delete(s.per, key)
		}
	}
	s.perMu.Unlock()
	select {
	case <-s.global:
	default:
	}
}

func capDigestFallback(content string, warn []string) (string, []string) {
	if digest.EstTokens(content) <= digest.SmallDoc {
		return content, warn
	}
	return digest.CapBudget(content), append(warn, "clean fallback truncated to digest budget")
}

func hostOf(raw string) string {
	u, err := url.Parse(raw)
	if err != nil {
		return ""
	}
	return u.Hostname()
}

func permittedCaller(name string, permit []string) bool {
	if name == "" {
		return false
	}
	for _, p := range permit {
		if p == name {
			return true
		}
	}
	return false
}

// logHost is the host field for request logs. permit_restricted callers get "-".
func (s *Server) logHost(c *auth.Caller, raw string) string {
	if c != nil && permittedCaller(c.Name, s.PermitRestricted) {
		return "-"
	}
	return hostOf(raw)
}

func (s *Server) restrictedLog(c *auth.Caller) bool {
	return c != nil && permittedCaller(c.Name, s.PermitRestricted)
}

func (s *Server) callerClass(c *auth.Caller) string {
	return dcache.CallerClass(s.restrictedLog(c))
}

// logReason is the reason field for fetch error logs. Restricted callers get a
// coarse class so the hostname in guard messages does not land in the journal.
func (s *Server) logReason(c *auth.Caller, fe *fetch.Error) string {
	if fe == nil {
		return "-"
	}
	if s.restrictedLog(c) {
		return coarseReason(fe)
	}
	return fe.Msg
}

func (s *Server) logErr(c *auth.Caller, err error) any {
	if s.restrictedLog(c) {
		return "-"
	}
	return err
}

func coarseReason(fe *fetch.Error) string {
	switch fe.Status {
	case 401:
		return "unauthorized"
	case 403:
		return "blocked"
	case 400:
		return "bad request"
	case 413:
		return "too large"
	case 415:
		return "unsupported"
	case 429:
		return "busy"
	case 504:
		return "timeout"
	default:
		if fe.Status >= 400 && fe.Status < 500 {
			return "upstream 4xx"
		}
		if fe.Status >= 500 {
			return "upstream 5xx"
		}
		return "-"
	}
}

func writeErr(w http.ResponseWriter, status int, msg, hint string) {
	if status == 502 {
		status = 500
		msg = "internal error"
	}
	body := map[string]any{"success": false, "error": msg}
	if hint != "" {
		body["hint"] = hint
	}
	writeJSON(w, status, body)
}

func writeJSON(w http.ResponseWriter, status int, v any) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	_ = json.NewEncoder(w).Encode(v)
}

func HashBody(b []byte) string {
	sum := sha256.Sum256(b)
	return hex.EncodeToString(sum[:])
}

func Drain(r io.Reader) {
	_, _ = io.Copy(io.Discard, r)
}