Add mcp-agent recover command

Recreates containers from the agent's SQLite registry when podman's
database is lost (UID change, podman reset, reboot). For each service
with desired_state="running" that doesn't have a running container:

- Removes any stale container with the same name
- Recreates the container from the stored spec (image, ports, volumes,
  cmd, network, user, restart policy)
- Allocates route ports and injects PORT env vars
- Re-registers mc-proxy routes
- Provisions TLS certs for L7 routes

Does NOT pull images — assumes local cache.

Root cause action item from the 2026-04-03 UID incident.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

cmd/mcp-agent/main.go

@@ -43,6 +43,7 @@ func main() {
 	})
 
 	root.AddCommand(snapshotCmd())
+	root.AddCommand(recoverCmd())
 
 	if err := root.Execute(); err != nil {
 		log.Fatal(err)

cmd/mcp-agent/recover.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+
+	"git.wntrmute.dev/mc/mcp/internal/agent"
+	"git.wntrmute.dev/mc/mcp/internal/config"
+	"git.wntrmute.dev/mc/mcp/internal/registry"
+	"git.wntrmute.dev/mc/mcp/internal/runtime"
+	"github.com/spf13/cobra"
+)
+
+func recoverCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "recover",
+		Short: "Recreate containers from the agent registry",
+		Long: `Recover recreates containers from the agent's SQLite registry for all
+services whose desired state is "running" but which don't have a running
+container in podman.
+
+This is the recovery path after a podman database loss (e.g., after a
+UID change, podman reset, or reboot that cleared container state).
+
+Images must be cached locally — recover does not pull from MCR.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.LoadAgentConfig(cfgPath)
+			if err != nil {
+				return fmt.Errorf("load config: %w", err)
+			}
+
+			logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+				Level: slog.LevelInfo,
+			}))
+
+			db, err := registry.Open(cfg.Database.Path)
+			if err != nil {
+				return fmt.Errorf("open registry: %w", err)
+			}
+			defer func() { _ = db.Close() }()
+
+			proxy, err := agent.NewProxyRouter(cfg.MCProxy.Socket, cfg.MCProxy.CertDir, logger)
+			if err != nil {
+				logger.Warn("mc-proxy not available, routes will not be registered", "err", err)
+			}
+
+			certs, err := agent.NewCertProvisioner(cfg.Metacrypt, cfg.MCProxy.CertDir, logger)
+			if err != nil {
+				logger.Warn("cert provisioner not available", "err", err)
+			}
+
+			a := &agent.Agent{
+				Config:    cfg,
+				DB:        db,
+				Runtime:   &runtime.Podman{},
+				Logger:    logger,
+				PortAlloc: agent.NewPortAllocator(),
+				Proxy:     proxy,
+				Certs:     certs,
+				Version:   version,
+			}
+
+			return a.Recover(context.Background())
+		},
+	}
+}

cmd/mcp/dial.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	mcpv1 "git.wntrmute.dev/mc/mcp/gen/mcp/v1"
 	"git.wntrmute.dev/mc/mcp/internal/config"
@@ -52,6 +53,38 @@ func dialAgent(address string, cfg *config.CLIConfig) (mcpv1.McpAgentServiceClie
 	return mcpv1.NewMcpAgentServiceClient(conn), conn, nil
 }
 
+// dialAgentMulti tries each address in order and returns the first successful
+// connection. Provides resilience when Tailscale DNS is down or a node is
+// reachable via LAN but not Tailnet.
+func dialAgentMulti(addresses []string, cfg *config.CLIConfig) (mcpv1.McpAgentServiceClient, *grpc.ClientConn, error) {
+	if len(addresses) == 0 {
+		return nil, nil, fmt.Errorf("no addresses to dial")
+	}
+	if len(addresses) == 1 {
+		return dialAgent(addresses[0], cfg)
+	}
+
+	var lastErr error
+	for _, addr := range addresses {
+		client, conn, err := dialAgent(addr, cfg)
+		if err != nil {
+			lastErr = fmt.Errorf("%s: %w", addr, err)
+			continue
+		}
+		// Quick health check to verify the connection actually works.
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		_, err = client.NodeStatus(ctx, &mcpv1.NodeStatusRequest{})
+		cancel()
+		if err != nil {
+			_ = conn.Close()
+			lastErr = fmt.Errorf("%s: %w", addr, err)
+			continue
+		}
+		return client, conn, nil
+	}
+	return nil, nil, fmt.Errorf("all addresses failed, last error: %w", lastErr)
+}
+
 // dialMaster connects to the master at the given address and returns a gRPC
 // client for the McpMasterService.
 func dialMaster(address string, cfg *config.CLIConfig) (mcpv1.McpMasterServiceClient, *grpc.ClientConn, error) {

cmd/mcp/helpers.go

@@ -9,7 +9,7 @@ import (
 )
 
 // findNodeAddress looks up a node by name in the CLI config and returns
-// its address.
+// its primary address.
 func findNodeAddress(cfg *config.CLIConfig, nodeName string) (string, error) {
 	for _, n := range cfg.Nodes {
 		if n.Name == nodeName {
@@ -19,6 +19,16 @@ func findNodeAddress(cfg *config.CLIConfig, nodeName string) (string, error) {
 	return "", fmt.Errorf("node %q not found in config", nodeName)
 }
 
+// findNode looks up a node by name in the CLI config.
+func findNode(cfg *config.CLIConfig, nodeName string) (*config.NodeConfig, error) {
+	for i := range cfg.Nodes {
+		if cfg.Nodes[i].Name == nodeName {
+			return &cfg.Nodes[i], nil
+		}
+	}
+	return nil, fmt.Errorf("node %q not found in config", nodeName)
+}
+
 // printComponentResults prints the result of each component operation.
 func printComponentResults(results []*mcpv1.ComponentResult) {
 	for _, r := range results {

cmd/mcp/status.go

@@ -27,7 +27,7 @@ func forEachNode(fn func(node config.NodeConfig, client mcpv1.McpAgentServiceCli
 	}
 
 	for _, node := range cfg.Nodes {
-		client, conn, err := dialAgent(node.Address, cfg)
+		client, conn, err := dialAgentMulti(node.AllAddresses(), cfg)
 		if err != nil {
 			_, _ = fmt.Fprintf(os.Stderr, "warning: %s: %v\n", node.Name, err)
 			continue
@@ -85,7 +85,7 @@ func psCmd() *cobra.Command {
 		Short: "Live check: query runtime on all agents",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			w := newTable()
-			_, _ = fmt.Fprintln(w, "SERVICE\tCOMPONENT\tNODE\tSTATE\tVERSION\tUPTIME")
+			_, _ = fmt.Fprintln(w, "SERVICE\tCOMPONENT\tNODE\tSTATE\tVERSION\tUPTIME\t")
 
 			now := time.Now()
 			if err := forEachNode(func(node config.NodeConfig, client mcpv1.McpAgentServiceClient) error {
@@ -96,19 +96,25 @@ func psCmd() *cobra.Command {
 				}
 
 				for _, svc := range resp.GetServices() {
+					comment := svc.GetComment()
 					for _, comp := range svc.GetComponents() {
 						uptime := "-"
 						if comp.GetStarted() != nil {
 							d := now.Sub(comp.GetStarted().AsTime())
 							uptime = formatDuration(d)
 						}
-						_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
+						col7 := ""
+						if comment != "" {
+							col7 = "# " + comment
+						}
+						_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
 							svc.GetName(),
 							comp.GetName(),
 							node.Name,
 							comp.GetObservedState(),
 							comp.GetVersion(),
 							uptime,
+							col7,
 						)
 					}
 				}

internal/agent/recover.go

@@ -0,0 +1,139 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+
+	"git.wntrmute.dev/mc/mcp/internal/registry"
+	"git.wntrmute.dev/mc/mcp/internal/runtime"
+)
+
+// Recover recreates containers from the agent's registry for all services
+// whose desired state is "running" but which don't have a running container
+// in podman. This is the recovery path after a podman database loss (e.g.,
+// after a UID change or podman reset).
+//
+// Recover does NOT pull images — it assumes the images are cached locally.
+// If an image is missing, that component is skipped with a warning.
+func (a *Agent) Recover(ctx context.Context) error {
+	services, err := registry.ListServices(a.DB)
+	if err != nil {
+		return fmt.Errorf("list services: %w", err)
+	}
+
+	// Get the list of currently running containers from podman.
+	running, err := a.Runtime.List(ctx)
+	if err != nil {
+		a.Logger.Warn("cannot list containers, assuming none running", "err", err)
+		running = nil
+	}
+	runningSet := make(map[string]bool)
+	for _, c := range running {
+		runningSet[c.Name] = true
+	}
+
+	var recovered, skipped, already int
+
+	for _, svc := range services {
+		if !svc.Active {
+			continue
+		}
+
+		comps, err := registry.ListComponents(a.DB, svc.Name)
+		if err != nil {
+			a.Logger.Warn("list components", "service", svc.Name, "err", err)
+			continue
+		}
+
+		for _, comp := range comps {
+			if comp.DesiredState != "running" {
+				continue
+			}
+
+			containerName := svc.Name + "-" + comp.Name
+			if comp.Name == svc.Name {
+				containerName = svc.Name
+			}
+
+			// Skip if container is already running.
+			if runningSet[containerName] {
+				already++
+				continue
+			}
+
+			a.Logger.Info("recovering container",
+				"service", svc.Name,
+				"component", comp.Name,
+				"image", comp.Image,
+			)
+
+			// Remove any stale container with the same name.
+			_ = a.Runtime.Remove(ctx, containerName)
+
+			// Build the container spec from the registry.
+			spec := runtime.ContainerSpec{
+				Name:    containerName,
+				Image:   comp.Image,
+				Network: comp.Network,
+				User:    comp.UserSpec,
+				Restart: comp.Restart,
+				Volumes: comp.Volumes,
+				Cmd:     comp.Cmd,
+			}
+
+			// Allocate ports from routes if the component has routes.
+			if len(comp.Routes) > 0 && a.PortAlloc != nil {
+				ports, env, allocErr := a.allocateRoutePorts(svc.Name, comp.Name, comp.Routes)
+				if allocErr != nil {
+					a.Logger.Warn("allocate route ports", "container", containerName, "err", allocErr)
+					spec.Ports = comp.Ports
+				} else {
+					spec.Ports = append(comp.Ports, ports...)
+					spec.Env = append(spec.Env, env...)
+				}
+			} else {
+				spec.Ports = comp.Ports
+			}
+
+			if err := a.Runtime.Run(ctx, spec); err != nil {
+				a.Logger.Error("recover container failed",
+					"container", containerName,
+					"err", err,
+				)
+				skipped++
+				continue
+			}
+
+			// Re-register mc-proxy routes.
+			if a.Proxy != nil && len(comp.Routes) > 0 {
+				hostPorts, hpErr := registry.GetRouteHostPorts(a.DB, svc.Name, comp.Name)
+				if hpErr == nil {
+					if proxyErr := a.Proxy.RegisterRoutes(ctx, svc.Name, comp.Routes, hostPorts); proxyErr != nil {
+						a.Logger.Warn("re-register routes", "service", svc.Name, "err", proxyErr)
+					}
+				}
+			}
+
+			// Provision TLS certs if needed.
+			if a.Certs != nil && hasL7Routes(comp.Routes) {
+				hostnames := l7Hostnames(svc.Name, comp.Routes)
+				if certErr := a.Certs.EnsureCert(ctx, svc.Name, hostnames); certErr != nil {
+					a.Logger.Warn("cert provisioning", "service", svc.Name, "err", certErr)
+				}
+			}
+
+			recovered++
+			a.Logger.Info("container recovered", "container", containerName)
+		}
+	}
+
+	a.Logger.Info("recovery complete",
+		"recovered", recovered,
+		"skipped", skipped,
+		"already_running", already,
+	)
+
+	return nil
+}
+
+// hasL7Routes and l7Hostnames are defined in deploy.go.

internal/config/cli.go

@@ -50,9 +50,28 @@ type AuthConfig struct {
 }
 
 // NodeConfig defines a managed node that the CLI connects to.
+// Address is the primary address. Addresses is an optional list of
+// fallback addresses tried in order if the primary fails. This
+// provides resilience when Tailscale DNS is down or a node is
+// reachable via LAN but not Tailnet.
 type NodeConfig struct {
 	Name      string   `toml:"name"`
 	Address   string   `toml:"address"`
+	Addresses []string `toml:"addresses,omitempty"`
+}
+
+// AllAddresses returns the node's primary address followed by any
+// fallback addresses, deduplicated.
+func (n NodeConfig) AllAddresses() []string {
+	seen := make(map[string]bool)
+	var addrs []string
+	for _, a := range append([]string{n.Address}, n.Addresses...) {
+		if a != "" && !seen[a] {
+			seen[a] = true
+			addrs = append(addrs, a)
+		}
+	}
+	return addrs
 }
 
 // LoadCLIConfig reads and validates a CLI configuration file.

internal/config/master.go

@@ -64,9 +64,24 @@ type TimeoutsConfig struct {
 type MasterNodeConfig struct {
 	Name      string   `toml:"name"`
 	Address   string   `toml:"address"`
+	Addresses []string `toml:"addresses,omitempty"`
 	Role      string   `toml:"role"` // "worker", "edge", or "master"
 }
 
+// AllAddresses returns the node's primary address followed by any
+// fallback addresses, deduplicated.
+func (n MasterNodeConfig) AllAddresses() []string {
+	seen := make(map[string]bool)
+	var addrs []string
+	for _, a := range append([]string{n.Address}, n.Addresses...) {
+		if a != "" && !seen[a] {
+			seen[a] = true
+			addrs = append(addrs, a)
+		}
+	}
+	return addrs
+}
+
 // LoadMasterConfig reads and validates a master configuration file.
 func LoadMasterConfig(path string) (*MasterConfig, error) {
 	data, err := os.ReadFile(path) //nolint:gosec // config path from trusted CLI flag

internal/master/agentclient.go

@@ -140,21 +140,30 @@ func NewAgentPool(caCertPath, token string) *AgentPool {
 
 // AddNode dials an agent and adds it to the pool.
 func (p *AgentPool) AddNode(name, address string) error {
-	client, err := DialAgent(address, p.caCert, p.token)
+	return p.AddNodeMulti(name, []string{address})
+}
+
+// AddNodeMulti tries each address in order and adds the first successful
+// connection to the pool.
+func (p *AgentPool) AddNodeMulti(name string, addresses []string) error {
+	var lastErr error
+	for _, addr := range addresses {
+		client, err := DialAgent(addr, p.caCert, p.token)
 		if err != nil {
-		return fmt.Errorf("add node %s: %w", name, err)
+			lastErr = fmt.Errorf("%s: %w", addr, err)
+			continue
 		}
 		client.Node = name
 
 		p.mu.Lock()
-	defer p.mu.Unlock()
-
-	// Close existing connection if re-adding.
 		if old, ok := p.clients[name]; ok {
 			_ = old.Close()
 		}
 		p.clients[name] = client
+		p.mu.Unlock()
 		return nil
+	}
+	return fmt.Errorf("add node %s: all addresses failed: %w", name, lastErr)
 }
 
 // Get returns the agent client for a node.

internal/master/master.go

@@ -63,7 +63,7 @@ func Run(cfg *config.MasterConfig, version string) error {
 	// Create agent connection pool.
 	pool := NewAgentPool(cfg.Master.CACert, token)
 	for _, n := range cfg.Nodes {
-		if addErr := pool.AddNode(n.Name, n.Address); addErr != nil {
+		if addErr := pool.AddNodeMulti(n.Name, n.AllAddresses()); addErr != nil {
 			logger.Warn("failed to connect to agent", "node", n.Name, "err", addErr)
 			// Non-fatal: the node may come up later.
 		}