cmd/data_sync: update README

- Support suppressing console output.
- DefaultDebugOptions sets the correct tag now.
- FatalError(error, string) calls log.Fatal(message, err) if err != nil.

README.md

@@ -27,6 +27,7 @@ Contents:
         cruntar/    Untar an archive with hard links, copying instead of
                     linking.
         csrpubdump/ Dump the public key from an X.509 certificate request.
+        data_sync/  Sync the user's homedir to external storage.
         diskimg/    Write a disk image to a device.
         eig/        EEPROM image generator.
         fragment/   Print a fragment of a file.

cmd/data_sync/README

@@ -0,0 +1,32 @@
+data_sync
+
+This is a tool I wrote primarily to sync my home directory to a backup
+drive plugged into my laptop. This system is provisioned by Ansible,
+and the goal is to be able to just copy my home directory back in the
+event of a failure without having lost a great deal of work or to wait
+for ansible to finish installing the right backup software. Specifically,
+I use a Framework laptop with the 1TB storage module, encrypted with
+LUKS, and run this twice daily (timed to correspond with my commute,
+though that's not really necessary). It started off as a shell script,
+then I decided to just write it as a program.
+
+Usage: data_sync [-d path] [-l level] [-m path] [-nqsv]
+                                  [-t path]
+        -d path         path to sync source directory
+                        (default "~")
+        -l level        log level to output (default "INFO"). Valid log
+                        levels are DEBUG, INFO, NOTICE, WARNING, ERR,
+                        CRIT, ALERT, EMERG. The default is INFO.
+        -m path         path to sync mount directory
+                        (default "/media/$USER/$(hostname -s)_data")
+        -n              dry-run mode: only check paths and print files to
+                        exclude
+        -q              suppress console output
+        -s              suppress syslog output
+        -t path         path to sync target directory
+                        (default "/media/$USER/$(hostname -s)_data/$USER")
+        -v              verbose rsync output
+
+data_sync rsyncs the tree at the sync source directory (-d) to the sync target
+directory (-t); it checks the mount directory (-m) exists; the sync target
+target directory must exist on the mount directory.

cmd/data_sync/main.go

@@ -0,0 +1,230 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"git.wntrmute.dev/kyle/goutils/config"
+	"git.wntrmute.dev/kyle/goutils/fileutil"
+	"git.wntrmute.dev/kyle/goutils/log"
+)
+
+func mustHostname() string {
+	hostname, err := os.Hostname()
+	log.FatalError(err, "couldn't retrieve hostname")
+
+	if hostname == "" {
+		log.Fatal("no hostname returned")
+	}
+	return strings.Split(hostname, ".")[0]
+}
+
+var (
+	defaultDataDir   = mustHostname() + "_data"
+	defaultProgName  = defaultDataDir + "_sync"
+	defaultMountDir  = filepath.Join("/media", os.Getenv("USER"), defaultDataDir)
+	defaultSyncDir   = os.Getenv("HOME")
+	defaultTargetDir = filepath.Join(defaultMountDir, os.Getenv("USER"))
+)
+
+func usage(w io.Writer) {
+	prog := filepath.Base(os.Args[0])
+	fmt.Fprintf(w, `Usage: %s [-d path] [-l level] [-m path] [-nqsv]
+				  [-t path]
+	-d path		path to sync source directory
+			(default "%s")
+	-l level	log level to output (default "INFO"). Valid log
+			levels are DEBUG, INFO, NOTICE, WARNING, ERR,
+			CRIT, ALERT, EMERG. The default is INFO.
+	-m path		path to sync mount directory
+			(default "%s")
+	-n		dry-run mode: only check paths and print files to
+			exclude
+	-q		suppress console output
+	-s		suppress syslog output
+	-t path		path to sync target directory
+			(default "%s")
+	-v		verbose rsync output
+
+%s rsyncs the tree at the sync source directory (-d) to the sync target
+directory (-t); it checks the mount directory (-m) exists; the sync target
+target directory must exist on the mount directory.
+
+`, prog, defaultSyncDir, defaultMountDir, defaultTargetDir, prog)
+}
+
+func checkPaths(mount, target string, dryRun bool) error {
+	if !fileutil.DirectoryDoesExist(mount) {
+		return fmt.Errorf("sync dir %s isn't mounted", mount)
+	}
+
+	if !strings.HasPrefix(target, mount) {
+		return fmt.Errorf("target dir %s must exist in %s", target, mount)
+	}
+
+	if !fileutil.DirectoryDoesExist(target) {
+		if dryRun {
+			log.Infof("would mkdir %s", target)
+		} else {
+			log.Infof("mkdir %s", target)
+			if err := os.Mkdir(target, 0755); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func buildExcludes(syncDir string) ([]string, error) {
+	var excluded []string
+
+	walker := func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			excluded = append(excluded, strings.TrimPrefix(path, syncDir))
+			if info != nil && info.IsDir() {
+				return filepath.SkipDir
+			}
+
+			return nil
+		}
+
+		if info.Mode().IsRegular() {
+			if err = fileutil.Access(path, fileutil.AccessRead); err != nil {
+				excluded = append(excluded, strings.TrimPrefix(path, syncDir))
+			}
+		}
+
+		if info.IsDir() {
+			if err = fileutil.Access(path, fileutil.AccessExec); err != nil {
+				excluded = append(excluded, strings.TrimPrefix(path, syncDir))
+			}
+		}
+
+		return nil
+	}
+
+	err := filepath.Walk(syncDir, walker)
+	return excluded, err
+}
+
+func writeExcludes(excluded []string) (string, error) {
+	if len(excluded) == 0 {
+		return "", nil
+	}
+
+	excludeFile, err := os.CreateTemp("", defaultProgName)
+	if err != nil {
+		return "", err
+	}
+
+	for _, name := range excluded {
+		fmt.Fprintln(excludeFile, name)
+	}
+
+	defer excludeFile.Close()
+	return excludeFile.Name(), nil
+}
+
+func rsync(syncDir, target, excludeFile string, verboseRsync bool) error {
+	var args []string
+
+	if excludeFile != "" {
+		args = append(args, "--exclude-from")
+		args = append(args, excludeFile)
+	}
+
+	if verboseRsync {
+		args = append(args, "--progress")
+		args = append(args, "-v")
+	}
+
+	args = append(args, []string{"-au", syncDir + "/", target + "/"}...)
+
+	path, err := exec.LookPath("rsync")
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command(path, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func init() {
+	flag.Usage = func() { usage(os.Stderr) }
+}
+
+func main() {
+
+	var logLevel, mountDir, syncDir, target string
+	var dryRun, quietMode, noSyslog, verboseRsync bool
+
+	flag.StringVar(&syncDir, "d", config.GetDefault("sync_dir", defaultSyncDir),
+		"`path to sync source directory`")
+	flag.StringVar(&logLevel, "l", config.GetDefault("log_level", "INFO"),
+		"log level to output")
+	flag.StringVar(&mountDir, "m", config.GetDefault("mount_dir", defaultMountDir),
+		"`path` to sync mount directory")
+	flag.BoolVar(&dryRun, "n", false, "dry-run mode: only check paths and print files to exclude")
+	flag.BoolVar(&quietMode, "q", quietMode, "suppress console output")
+	flag.BoolVar(&noSyslog, "s", noSyslog, "suppress syslog output")
+	flag.StringVar(&target, "t", config.GetDefault("sync_target", defaultTargetDir),
+		"`path` to sync target directory")
+	flag.BoolVar(&verboseRsync, "v", false, "verbose rsync output")
+	flag.Parse()
+
+	if quietMode && noSyslog {
+		fmt.Fprintln(os.Stderr, "both console and syslog output are suppressed")
+		fmt.Fprintln(os.Stderr, "errors will NOT be reported")
+	}
+
+	logOpts := &log.Options{
+		Level:        logLevel,
+		Tag:          defaultProgName,
+		Facility:     "user",
+		WriteSyslog:  !noSyslog,
+		WriteConsole: !quietMode,
+	}
+	err := log.Setup(logOpts)
+	log.FatalError(err, "failed to set up logging")
+
+	log.Infof("checking paths: mount=%s, target=%s", mountDir, target)
+	err = checkPaths(mountDir, target, dryRun)
+	log.FatalError(err, "target dir isn't ready")
+
+	log.Infof("checking for files to exclude from %s", syncDir)
+	excluded, err := buildExcludes(syncDir)
+	log.FatalError(err, "couldn't build excludes")
+
+	if dryRun {
+		fmt.Println("excluded files:")
+		for _, path := range excluded {
+			fmt.Printf("\t%s\n", path)
+		}
+		return
+	}
+
+	excludeFile, err := writeExcludes(excluded)
+	log.FatalError(err, "couldn't write exclude file")
+	log.Infof("excluding %d files via %s", len(excluded), excludeFile)
+
+	if excludeFile != "" {
+		defer func() {
+			log.Infof("removing exclude file %s", excludeFile)
+			if err := os.Remove(excludeFile); err != nil {
+				log.Warningf("failed to remove temp file %s", excludeFile)
+			}
+		}()
+	}
+
+	err = rsync(syncDir, target, excludeFile, verboseRsync)
+	log.FatalError(err, "couldn't sync data")
+}

dbg/BUILD.bazel

@@ -13,7 +13,7 @@ go_test(
     srcs = ["dbg_test.go"],
     embed = [":dbg"],
     deps = [
+        "//assert",
         "//testio",
-        "@com_github_stretchr_testify//require",
     ],
 )

dbg/dbg_test.go

@@ -1,12 +1,13 @@
 package dbg
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
 
+	"git.wntrmute.dev/kyle/goutils/assert"
 	"git.wntrmute.dev/kyle/goutils/testio"
-	"github.com/stretchr/testify/require"
 )
 
 func TestNew(t *testing.T) {
@@ -17,16 +18,16 @@ func TestNew(t *testing.T) {
 	dbg.Print("hello")
 	dbg.Println("hello")
 	dbg.Printf("hello %s", "world")
-	require.Equal(t, 0, buf.Len())
+	assert.BoolT(t, buf.Len() == 0)
 
 	dbg.Enabled = true
 	dbg.Print("hello")              // +5
 	dbg.Println("hello")            // +6
 	dbg.Printf("hello %s", "world") // +11
-	require.Equal(t, 22, buf.Len())
+	assert.BoolT(t, buf.Len() == 22, fmt.Sprintf("buffer should be length 22 but is length %d", buf.Len()))
 
 	err := dbg.Close()
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 }
 
 func TestTo(t *testing.T) {
@@ -36,39 +37,38 @@ func TestTo(t *testing.T) {
 	dbg.Print("hello")
 	dbg.Println("hello")
 	dbg.Printf("hello %s", "world")
-	require.Equal(t, 0, buf.Len())
+	assert.BoolT(t, buf.Len() == 0, "debug output should be suppressed")
 
 	dbg.Enabled = true
 	dbg.Print("hello")              // +5
 	dbg.Println("hello")            // +6
 	dbg.Printf("hello %s", "world") // +11
-
-	require.Equal(t, 22, buf.Len())
+	assert.BoolT(t, buf.Len() == 22, "didn't get the expected debug output")
 
 	err := dbg.Close()
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 }
 
 func TestToFile(t *testing.T) {
 	testFile, err := ioutil.TempFile("", "dbg")
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 	err = testFile.Close()
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 
 	testFileName := testFile.Name()
 	defer os.Remove(testFileName)
 
 	dbg, err := ToFile(testFileName)
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 
 	dbg.Print("hello")
 	dbg.Println("hello")
 	dbg.Printf("hello %s", "world")
 
 	stat, err := os.Stat(testFileName)
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 
-	require.EqualValues(t, 0, stat.Size())
+	assert.BoolT(t, stat.Size() == 0, "no debug output should have been sent to the log file")
 
 	dbg.Enabled = true
 	dbg.Print("hello")              // +5
@@ -76,12 +76,12 @@ func TestToFile(t *testing.T) {
 	dbg.Printf("hello %s", "world") // +11
 
 	stat, err = os.Stat(testFileName)
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 
-	require.EqualValues(t, 22, stat.Size())
+	assert.BoolT(t, stat.Size() == 22, fmt.Sprintf("have %d bytes in the log file, expected 22", stat.Size()))
 
 	err = dbg.Close()
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 }
 
 func TestWriting(t *testing.T) {
@@ -90,31 +90,31 @@ func TestWriting(t *testing.T) {
 	dbg := To(buf)
 
 	n, err := dbg.Write(data)
-	require.NoError(t, err)
-	require.EqualValues(t, 0, n)
+	assert.NoErrorT(t, err)
+	assert.BoolT(t, n == 0, "expected nothing to be written to the buffer")
 
 	dbg.Enabled = true
 	n, err = dbg.Write(data)
-	require.NoError(t, err)
-	require.EqualValues(t, 12, n)
+	assert.NoErrorT(t, err)
+	assert.BoolT(t, n == 12, fmt.Sprintf("wrote %d bytes in the buffer, expected to write 12", n))
 
 	err = dbg.Close()
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 }
 
 func TestToFileError(t *testing.T) {
 	testFile, err := ioutil.TempFile("", "dbg")
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 	err = testFile.Chmod(0400)
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 	err = testFile.Close()
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 
 	testFileName := testFile.Name()
 
 	_, err = ToFile(testFileName)
-	require.Error(t, err)
+	assert.ErrorT(t, err)
 
 	err = os.Remove(testFileName)
-	require.NoError(t, err)
+	assert.NoErrorT(t, err)
 }

deps.bzl

@@ -221,8 +221,8 @@ def go_dependencies():
     go_repository(
         name = "com_github_stretchr_objx",
         importpath = "github.com/stretchr/objx",
-        sum = "h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=",
-        version = "v0.1.1",
+        sum = "h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=",
+        version = "v0.1.0",
     )
     go_repository(
         name = "com_github_stretchr_testify",
@@ -302,12 +302,6 @@ def go_dependencies():
         sum = "h1:bkb2NMGo3/Du52wvYj9Whth5KZfMV6d3O0Vbr3nz/UE=",
         version = "v0.0.0-20150115234039-8488cc47d90c",
     )
-    go_repository(
-        name = "org_golang_google_appengine",
-        importpath = "google.golang.org/appengine",
-        sum = "h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=",
-        version = "v1.6.6",
-    )
     go_repository(
         name = "org_golang_x_crypto",
         importpath = "golang.org/x/crypto",

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/kr/text v0.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.12.0
-	github.com/stretchr/testify v1.6.1
 	golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b
 	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
 	gopkg.in/yaml.v2 v2.4.0
@@ -21,7 +20,5 @@ require (
 require (
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )

log/logger.go

@@ -12,8 +12,9 @@ import (
 )
 
 type logger struct {
-	l gsyslog.Syslogger
-	p gsyslog.Priority
+	l            gsyslog.Syslogger
+	p            gsyslog.Priority
+	writeConsole bool
 }
 
 func (log *logger) printf(p gsyslog.Priority, format string, args ...interface{}) {
@@ -21,7 +22,7 @@ func (log *logger) printf(p gsyslog.Priority, format string, args ...interface{}
 		format += "\n"
 	}
 
-	if p <= log.p {
+	if p <= log.p && log.writeConsole {
 		fmt.Printf("%s [%s] ", prioritiev[p], timestamp())
 		fmt.Printf(format, args...)
 	}
@@ -32,7 +33,7 @@ func (log *logger) printf(p gsyslog.Priority, format string, args ...interface{}
 }
 
 func (log *logger) print(p gsyslog.Priority, args ...interface{}) {
-	if p <= log.p {
+	if p <= log.p && log.writeConsole {
 		fmt.Printf("%s [%s] ", prioritiev[p], timestamp())
 		fmt.Print(args...)
 	}
@@ -43,7 +44,7 @@ func (log *logger) print(p gsyslog.Priority, args ...interface{}) {
 }
 
 func (log *logger) println(p gsyslog.Priority, args ...interface{}) {
-	if p <= log.p {
+	if p <= log.p && log.writeConsole {
 		fmt.Printf("%s [%s] ", prioritiev[p], timestamp())
 		fmt.Println(args...)
 	}
@@ -98,10 +99,11 @@ func timestamp() string {
 }
 
 type Options struct {
-	Level       string
-	Tag         string
-	Facility    string
-	WriteSyslog bool
+	Level        string
+	Tag          string
+	Facility     string
+	WriteSyslog  bool
+	WriteConsole bool
 }
 
 // DefaultOptions returns a sane set of defaults for syslog, using the program
@@ -113,10 +115,11 @@ func DefaultOptions(tag string, withSyslog bool) *Options {
 	}
 
 	return &Options{
-		Level:       "WARNING",
-		Tag:         tag,
-		Facility:    "daemon",
-		WriteSyslog: withSyslog,
+		Level:        "WARNING",
+		Tag:          tag,
+		Facility:     "daemon",
+		WriteSyslog:  withSyslog,
+		WriteConsole: true,
 	}
 }
 
@@ -129,9 +132,11 @@ func DefaultDebugOptions(tag string, withSyslog bool) *Options {
 	}
 
 	return &Options{
-		Level:       "DEBUG",
-		Facility:    "daemon",
-		WriteSyslog: withSyslog,
+		Level:        "DEBUG",
+		Tag:          tag,
+		Facility:     "daemon",
+		WriteSyslog:  withSyslog,
+		WriteConsole: true,
 	}
 }
 
@@ -142,6 +147,10 @@ func Setup(opts *Options) error {
 	}
 
 	log.p = priority
+	log.writeConsole = opts.WriteConsole
+	if opts.WriteConsole {
+		fmt.Println("will write to console")
+	}
 
 	if opts.WriteSyslog {
 		var err error
@@ -261,6 +270,17 @@ func Fatalf(format string, args ...interface{}) {
 	os.Exit(1)
 }
 
+// FatalError will only execute if err != nil. If it does,
+// it will print the message (append the error) and exit
+// the program.
+func FatalError(err error, message string) {
+	if err == nil {
+		return
+	}
+
+	Fatal(fmt.Sprintf("%s: %s", message, err))
+}
+
 // Spew will pretty print the args if the logger is set to DEBUG priority.
 func Spew(args ...interface{}) {
 	log.spew(args...)