diff --git a/PROGRESS.md b/PROGRESS.md index 270660f..8ad1a74 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,59 +2,76 @@ ## Current State -Phase 3 complete. The `db`, `auth`, and `config` packages are implemented -and tested. +Phases 0–9 complete. All nine packages are implemented and tested (87 tests). +Ready for first-adopter migration (Phase 10). ## Completed ### Phase 0: Project Setup (2026-03-25) -- Initialized Go module (`git.wntrmute.dev/kyle/mcdsl`) -- Created `.golangci.yaml` matching platform standard (with `exported` rule - enabled since this is a shared library) -- Created `Makefile` with standard targets (build, test, vet, lint, all) -- Created `.gitignore` -- Created `doc.go` package doc -- `make all` passes clean +- Go module, Makefile, .golangci.yaml (with `exported` rule), .gitignore ### Phase 1: `db` — SQLite Foundation (2026-03-25) -- `Open(path string) (*sql.DB, error)` — opens with WAL, FK, busy timeout - 5000ms, 0600 permissions, creates parent dirs -- `Migration` type with Version, Name, SQL fields -- `Migrate(database *sql.DB, migrations []Migration) error` — sequential, - transactional, idempotent, records name and timestamp in schema_migrations -- `SchemaVersion(database *sql.DB) (int, error)` — highest applied version -- `Snapshot(database *sql.DB, destPath string) error` — VACUUM INTO with - 0600 permissions, creates parent dirs -- 11 tests covering open, migrate, and snapshot +- Open (WAL, FK, busy timeout, 0600, parent dirs), Migration type, Migrate + (sequential, transactional, idempotent), SchemaVersion, Snapshot (VACUUM INTO) +- 11 tests ### Phase 2: `auth` — MCIAS Token Validation (2026-03-25) -- `Config` type matching `[mcias]` TOML section -- `TokenInfo` type (Username, Roles, IsAdmin) -- `New(cfg, logger)` — MCIAS client with TLS 1.3, custom CA, 10s timeout -- `Login`, `ValidateToken` (30s SHA-256 cache), `Logout` -- Error types, context helpers -- 14 tests with mock MCIAS server and injectable clock +- Config, TokenInfo, Authenticator with Login/ValidateToken/Logout +- 30s SHA-256 cache, lazy eviction, RWMutex, context helpers +- 14 tests ### Phase 3: `config` — TOML Configuration (2026-03-25) -- `Base` type embedding standard sections (Server, Database, MCIAS, Log) -- `ServerConfig` with `Duration` wrapper type for TOML string decoding - (go-toml v2 does not natively decode strings to time.Duration) -- `DatabaseConfig`, `LogConfig`, `WebConfig` (non-embedded, for web UIs) -- `Duration` type with TextUnmarshaler/TextMarshaler for TOML compatibility -- `Load[T any](path, envPrefix)` — generic loader with TOML parse, env - overrides via reflection, defaults, required field validation -- `Validator` interface for service-specific validation -- Environment overrides: PREFIX_SECTION_FIELD for strings, durations, - bools, and comma-separated string slices -- Defaults: ReadTimeout=30s, WriteTimeout=30s, IdleTimeout=120s, - ShutdownTimeout=60s, Log.Level="info" -- Required: listen_addr, tls_cert, tls_key -- 16 tests: minimal/full config, defaults (applied and not overriding - explicit), missing required fields (3 cases), env overrides (string, - duration, slice, bool, service-specific), Validator interface (pass/fail), - nonexistent file, invalid TOML, empty prefix -- `make all` passes clean (vet, lint 0 issues, 41 total tests, build) +- Base type, ServerConfig with Duration wrapper, Load[T] generic loader +- Env overrides via reflection, defaults, Validator interface +- 16 tests + +### Phase 4: `httpserver` — HTTP Server (2026-03-25) +- Server with chi + TLS 1.3, ListenAndServeTLS, Shutdown +- LoggingMiddleware, StatusWriter, WriteJSON, WriteError +- 8 tests + +### Phase 5: `csrf` — CSRF Protection (2026-03-25) +- HMAC-SHA256 double-submit cookies, Middleware, SetToken, TemplateFunc +- 10 tests + +### Phase 6: `web` — Session and Templates (2026-03-25) +- SetSessionCookie/ClearSessionCookie/GetSessionToken (HttpOnly, Secure, + SameSite=Strict), RequireAuth middleware, RenderTemplate +- 9 tests + +### Phase 7: `grpcserver` — gRPC Server (2026-03-25) +- MethodMap (Public, AuthRequired, AdminRequired), default deny for unmapped +- Auth interceptor, logging interceptor, TLS 1.3 optional +- 10 tests + +### Phase 8: `health` — Health Checks (2026-03-25) +- REST Handler(db) — 200 ok / 503 unhealthy +- RegisterGRPC — grpc.health.v1.Health +- 4 tests + +### Phase 9: `archive` — Service Directory Snapshots (2026-03-25) +- Snapshot: tar.zst with VACUUM INTO db injection, exclude *.db/*.db-wal/ + *.db-shm/backups/, custom exclude patterns, streaming output +- Restore: extract tar.zst to dest dir, path traversal protection +- 5 tests: full roundtrip with db integrity, without db, exclude live db, + custom excludes, dest dir creation + +## Summary + +| Package | Tests | Key Exports | +|---------|-------|-------------| +| `db` | 11 | Open, Migration, Migrate, SchemaVersion, Snapshot | +| `auth` | 14 | Config, TokenInfo, Authenticator, context helpers | +| `config` | 16 | Base, ServerConfig, Duration, Load[T], Validator | +| `httpserver` | 8 | Server, LoggingMiddleware, WriteJSON, WriteError | +| `csrf` | 10 | Protect, Middleware, SetToken, TemplateFunc | +| `web` | 9 | SetSessionCookie, RequireAuth, RenderTemplate | +| `grpcserver` | 10 | MethodMap, Server (default deny), TokenInfoFromContext | +| `health` | 4 | Handler, RegisterGRPC | +| `archive` | 5 | Snapshot, Restore | +| **Total** | **87** | | ## Next Steps -- Phase 4: `httpserver` package (TLS HTTP server, middleware, JSON helpers) +- Phase 10: First-adopter migration (mcat) +- Phase 11: Broader adoption (metacrypt, mcr, mc-proxy, mcias) diff --git a/archive/archive.go b/archive/archive.go new file mode 100644 index 0000000..00fde70 --- /dev/null +++ b/archive/archive.go @@ -0,0 +1,235 @@ +// Package archive provides service directory snapshot and restore using +// tar.zst (tar compressed with Zstandard), with SQLite-aware handling. +// +// Snapshots exclude live database files (*.db, *.db-wal, *.db-shm) and +// the backups/ directory. A consistent database copy is created via +// VACUUM INTO and injected into the archive as .db. +// +// The result is a clean, minimal archive that extracts directly into a +// working service directory on the destination. +package archive + +import ( + "archive/tar" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/klauspost/compress/zstd" + + "git.wntrmute.dev/kyle/mcdsl/db" +) + +// defaultExcludePatterns are always excluded from snapshots. +var defaultExcludePatterns = []string{ + "*.db", + "*.db-wal", + "*.db-shm", + "backups", +} + +// SnapshotOptions configures a snapshot operation. +type SnapshotOptions struct { + // ServiceDir is the root directory to snapshot (e.g., /srv/myservice). + ServiceDir string + + // DBPath is the path to the live database file. The filename (without + // directory) is used as the name in the archive. If empty, no database + // is included. + DBPath string + + // DB is the live database connection. Used for VACUUM INTO to create + // a consistent snapshot. Required if DBPath is set. + DB *sql.DB + + // ExcludePatterns are additional glob patterns to exclude (beyond the + // defaults: *.db, *.db-wal, *.db-shm, backups/). + ExcludePatterns []string +} + +// Snapshot creates a tar.zst archive of the service directory, writing +// it to w. Live database files are excluded and a consistent VACUUM INTO +// copy is injected in their place. +func Snapshot(opts SnapshotOptions, w io.Writer) error { + // Create the VACUUM INTO copy if a database is specified. + var dbSnapshotPath string + if opts.DBPath != "" && opts.DB != nil { + tmp, err := os.CreateTemp("", "mcdsl-snapshot-*.db") + if err != nil { + return fmt.Errorf("archive: create temp db: %w", err) + } + tmpPath := tmp.Name() + _ = tmp.Close() + _ = os.Remove(tmpPath) // VACUUM INTO creates the file itself + + if err := db.Snapshot(opts.DB, tmpPath); err != nil { + return fmt.Errorf("archive: vacuum into: %w", err) + } + dbSnapshotPath = tmpPath + defer func() { _ = os.Remove(tmpPath) }() + } + + // Build the exclude set. + excludes := append(defaultExcludePatterns, opts.ExcludePatterns...) + + // Create zstd writer → tar writer. + zw, err := zstd.NewWriter(w) + if err != nil { + return fmt.Errorf("archive: create zstd writer: %w", err) + } + + tw := tar.NewWriter(zw) + + // Walk the service directory. + serviceDir := filepath.Clean(opts.ServiceDir) + err = filepath.Walk(serviceDir, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + + relPath, relErr := filepath.Rel(serviceDir, path) + if relErr != nil { + return relErr + } + + // Skip the root directory itself. + if relPath == "." { + return nil + } + + // Check excludes. + if shouldExclude(relPath, info, excludes) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + return addToTar(tw, path, relPath, info) + }) + if err != nil { + return fmt.Errorf("archive: walk %s: %w", serviceDir, err) + } + + // Inject the database snapshot. + if dbSnapshotPath != "" { + dbName := filepath.Base(opts.DBPath) + snapInfo, statErr := os.Stat(dbSnapshotPath) + if statErr != nil { + return fmt.Errorf("archive: stat db snapshot: %w", statErr) + } + if err := addToTar(tw, dbSnapshotPath, dbName, snapInfo); err != nil { + return fmt.Errorf("archive: add db snapshot: %w", err) + } + } + + if err := tw.Close(); err != nil { + return fmt.Errorf("archive: close tar: %w", err) + } + if err := zw.Close(); err != nil { + return fmt.Errorf("archive: close zstd: %w", err) + } + + return nil +} + +// Restore extracts a tar.zst archive from r into destDir. Creates the +// directory if it does not exist. Overwrites existing files. Preserves +// file permissions. +func Restore(r io.Reader, destDir string) error { + if err := os.MkdirAll(destDir, 0700); err != nil { + return fmt.Errorf("archive: create dest dir: %w", err) + } + + zr, err := zstd.NewReader(r) + if err != nil { + return fmt.Errorf("archive: create zstd reader: %w", err) + } + defer zr.Close() + + tr := tar.NewReader(zr) + for { + header, readErr := tr.Next() + if readErr == io.EOF { + break + } + if readErr != nil { + return fmt.Errorf("archive: read tar entry: %w", readErr) + } + + target := filepath.Join(destDir, header.Name) //nolint:gosec // archive is from trusted MCP agent, not user upload + + // Prevent path traversal. + if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(destDir)+string(os.PathSeparator)) && + filepath.Clean(target) != filepath.Clean(destDir) { + return fmt.Errorf("archive: path traversal in %q", header.Name) + } + + switch header.Typeflag { + case tar.TypeDir: + if mkErr := os.MkdirAll(target, os.FileMode(header.Mode&0o777)); mkErr != nil { //nolint:gosec // mode from trusted archive + return fmt.Errorf("archive: mkdir %s: %w", target, mkErr) + } + case tar.TypeReg: + if mkErr := os.MkdirAll(filepath.Dir(target), 0700); mkErr != nil { + return fmt.Errorf("archive: mkdir parent %s: %w", target, mkErr) + } + f, createErr := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode&0o777)) //nolint:gosec // path validated above, mode from trusted archive + if createErr != nil { + return fmt.Errorf("archive: create %s: %w", target, createErr) + } + if _, copyErr := io.Copy(f, tr); copyErr != nil { //nolint:gosec // bounded by tar entry size + _ = f.Close() + return fmt.Errorf("archive: write %s: %w", target, copyErr) + } + _ = f.Close() + } + } + + return nil +} + +// shouldExclude returns true if the given path matches any exclude pattern. +func shouldExclude(relPath string, info os.FileInfo, patterns []string) bool { + name := filepath.Base(relPath) + for _, pattern := range patterns { + // Match directory names exactly (e.g., "backups"). + if info.IsDir() && name == pattern { + return true + } + // Match file patterns (e.g., "*.db"). + if matched, _ := filepath.Match(pattern, name); matched { + return true + } + } + return false +} + +// addToTar adds a file or directory to the tar writer. +func addToTar(tw *tar.Writer, srcPath, archiveName string, info os.FileInfo) error { + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = archiveName + + if err := tw.WriteHeader(header); err != nil { + return err + } + + if info.IsDir() { + return nil + } + + f, err := os.Open(srcPath) //nolint:gosec // path from controlled walk + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + _, err = io.Copy(tw, f) + return err +} diff --git a/archive/archive_test.go b/archive/archive_test.go new file mode 100644 index 0000000..3d2b651 --- /dev/null +++ b/archive/archive_test.go @@ -0,0 +1,252 @@ +package archive + +import ( + "bytes" + "database/sql" + "os" + "path/filepath" + "testing" + + "git.wntrmute.dev/kyle/mcdsl/db" +) + +// setupServiceDir creates a realistic /srv// directory. +func setupServiceDir(t *testing.T, database *sql.DB) string { + t.Helper() + dir := t.TempDir() + + // Config file. + writeFile(t, filepath.Join(dir, "service.toml"), "listen_addr = \":8443\"\n") + + // Certs directory. + certsDir := filepath.Join(dir, "certs") + if err := os.Mkdir(certsDir, 0700); err != nil { + t.Fatalf("mkdir certs: %v", err) + } + writeFile(t, filepath.Join(certsDir, "cert.pem"), "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n") + writeFile(t, filepath.Join(certsDir, "key.pem"), "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n") + + // Live database file (should be excluded). + writeFile(t, filepath.Join(dir, "service.db"), "live-db-data") + writeFile(t, filepath.Join(dir, "service.db-wal"), "wal-data") + writeFile(t, filepath.Join(dir, "service.db-shm"), "shm-data") + + // Backups directory (should be excluded). + backupsDir := filepath.Join(dir, "backups") + if err := os.Mkdir(backupsDir, 0700); err != nil { + t.Fatalf("mkdir backups: %v", err) + } + writeFile(t, filepath.Join(backupsDir, "old-backup.db"), "backup-data") + + return dir +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func openTestDB(t *testing.T) (*sql.DB, string) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "real.db") + database, err := db.Open(path) + if err != nil { + t.Fatalf("db.Open: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + + // Create some data to verify snapshot integrity. + if _, err := database.Exec("CREATE TABLE test (id INTEGER PRIMARY KEY, val TEXT)"); err != nil { + t.Fatalf("create table: %v", err) + } + if _, err := database.Exec("INSERT INTO test (val) VALUES ('snapshot-data')"); err != nil { + t.Fatalf("insert: %v", err) + } + + return database, path +} + +func TestSnapshotAndRestore(t *testing.T) { + database, dbPath := openTestDB(t) + serviceDir := setupServiceDir(t, database) + + // Snapshot. + var buf bytes.Buffer + err := Snapshot(SnapshotOptions{ + ServiceDir: serviceDir, + DBPath: dbPath, + DB: database, + }, &buf) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + if buf.Len() == 0 { + t.Fatal("archive is empty") + } + + // Restore to a new directory. + restoreDir := t.TempDir() + if err := Restore(&buf, restoreDir); err != nil { + t.Fatalf("Restore: %v", err) + } + + // Verify config file was restored. + content, err := os.ReadFile(filepath.Join(restoreDir, "service.toml")) //nolint:gosec // test code + if err != nil { + t.Fatalf("read config: %v", err) + } + if string(content) != "listen_addr = \":8443\"\n" { + t.Fatalf("config content = %q", string(content)) + } + + // Verify certs were restored. + if _, err := os.Stat(filepath.Join(restoreDir, "certs", "cert.pem")); err != nil { + t.Fatalf("cert.pem missing: %v", err) + } + if _, err := os.Stat(filepath.Join(restoreDir, "certs", "key.pem")); err != nil { + t.Fatalf("key.pem missing: %v", err) + } + + // Verify live DB files were excluded. + if _, err := os.Stat(filepath.Join(restoreDir, "service.db-wal")); !os.IsNotExist(err) { + t.Fatal("service.db-wal should not be in archive") + } + if _, err := os.Stat(filepath.Join(restoreDir, "service.db-shm")); !os.IsNotExist(err) { + t.Fatal("service.db-shm should not be in archive") + } + + // Verify backups were excluded. + if _, err := os.Stat(filepath.Join(restoreDir, "backups")); !os.IsNotExist(err) { + t.Fatal("backups/ should not be in archive") + } + + // Verify the VACUUM INTO snapshot was injected as the DB. + dbFile := filepath.Join(restoreDir, filepath.Base(dbPath)) + restoredDB, err := sql.Open("sqlite", dbFile) + if err != nil { + t.Fatalf("open restored db: %v", err) + } + defer func() { _ = restoredDB.Close() }() + + var val string + if err := restoredDB.QueryRow("SELECT val FROM test").Scan(&val); err != nil { + t.Fatalf("query restored db: %v", err) + } + if val != "snapshot-data" { + t.Fatalf("val = %q, want %q", val, "snapshot-data") + } +} + +func TestSnapshotWithoutDB(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "config.toml"), "test = true\n") + + var buf bytes.Buffer + err := Snapshot(SnapshotOptions{ + ServiceDir: dir, + }, &buf) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + // Restore and verify. + restoreDir := t.TempDir() + if err := Restore(&buf, restoreDir); err != nil { + t.Fatalf("Restore: %v", err) + } + + content, err := os.ReadFile(filepath.Join(restoreDir, "config.toml")) //nolint:gosec // test code + if err != nil { + t.Fatalf("read: %v", err) + } + if string(content) != "test = true\n" { + t.Fatalf("content = %q", string(content)) + } +} + +func TestSnapshotExcludesLiveDB(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "service.db"), "live") + writeFile(t, filepath.Join(dir, "service.db-wal"), "wal") + writeFile(t, filepath.Join(dir, "other.txt"), "keep") + + var buf bytes.Buffer + err := Snapshot(SnapshotOptions{ + ServiceDir: dir, + }, &buf) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + restoreDir := t.TempDir() + if err := Restore(&buf, restoreDir); err != nil { + t.Fatalf("Restore: %v", err) + } + + // other.txt should be present. + if _, err := os.Stat(filepath.Join(restoreDir, "other.txt")); err != nil { + t.Fatalf("other.txt missing: %v", err) + } + + // DB files should not. + if _, err := os.Stat(filepath.Join(restoreDir, "service.db")); !os.IsNotExist(err) { + t.Fatal("service.db should be excluded") + } + if _, err := os.Stat(filepath.Join(restoreDir, "service.db-wal")); !os.IsNotExist(err) { + t.Fatal("service.db-wal should be excluded") + } +} + +func TestSnapshotCustomExcludes(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "keep.txt"), "keep") + writeFile(t, filepath.Join(dir, "skip.log"), "skip") + + var buf bytes.Buffer + err := Snapshot(SnapshotOptions{ + ServiceDir: dir, + ExcludePatterns: []string{"*.log"}, + }, &buf) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + restoreDir := t.TempDir() + if err := Restore(&buf, restoreDir); err != nil { + t.Fatalf("Restore: %v", err) + } + + if _, err := os.Stat(filepath.Join(restoreDir, "keep.txt")); err != nil { + t.Fatal("keep.txt should be present") + } + if _, err := os.Stat(filepath.Join(restoreDir, "skip.log")); !os.IsNotExist(err) { + t.Fatal("skip.log should be excluded") + } +} + +func TestRestoreCreatesDestDir(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "file.txt"), "data") + + var buf bytes.Buffer + if err := Snapshot(SnapshotOptions{ServiceDir: dir}, &buf); err != nil { + t.Fatalf("Snapshot: %v", err) + } + + destDir := filepath.Join(t.TempDir(), "new", "nested", "dir") + if err := Restore(&buf, destDir); err != nil { + t.Fatalf("Restore: %v", err) + } + + content, err := os.ReadFile(filepath.Join(destDir, "file.txt")) //nolint:gosec // test code + if err != nil { + t.Fatalf("read: %v", err) + } + if string(content) != "data" { + t.Fatalf("content = %q", string(content)) + } +} diff --git a/go.mod b/go.mod index d070d8c..f2a40c2 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.7 require ( github.com/go-chi/chi/v5 v5.2.5 + github.com/klauspost/compress v1.18.5 github.com/pelletier/go-toml/v2 v2.3.0 google.golang.org/grpc v1.79.3 modernc.org/sqlite v1.47.0 diff --git a/go.sum b/go.sum index fd53a93..75ea248 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=