data_sync: sync homedir to external storage.
This commit is contained in:
parent
7a4e7977c3
commit
b9f69e4aa1
|
|
@ -0,0 +1,31 @@
|
||||||
|
data_sync
|
||||||
|
|
||||||
|
This is a tool I wrote primarily to sync my home directory to a backup drive plugged
|
||||||
|
into my laptop. This system is provisioned by Ansible, and the goal is to be able to
|
||||||
|
just copy my home directory back in the event of a failure without having lost a great
|
||||||
|
deal of work. Specifically, I use a Framework laptop with the 1TB storage module,
|
||||||
|
encrypted with LUKS, and run this twice daily (timed to correspond with my commute,
|
||||||
|
though that's not really necessary). It started off as a shell script, then I decided
|
||||||
|
to just write it as a program.
|
||||||
|
|
||||||
|
Usage: data_sync [-d path] [-l level] [-m path] [-nqsv]
|
||||||
|
[-t path]
|
||||||
|
-d path path to sync source directory
|
||||||
|
(default "~")
|
||||||
|
-l level log level to output (default "INFO"). Valid log
|
||||||
|
levels are DEBUG, INFO, NOTICE, WARNING, ERR,
|
||||||
|
CRIT, ALERT, EMERG. The default is INFO.
|
||||||
|
-m path path to sync mount directory
|
||||||
|
(default "/media/$USER/$(hostname -s)_data")
|
||||||
|
-n dry-run mode: only check paths and print files to
|
||||||
|
exclude
|
||||||
|
-q suppress console output
|
||||||
|
-s suppress syslog output
|
||||||
|
-t path path to sync target directory
|
||||||
|
(default "/media/$USER/$(hostname -s)_data/$USER")
|
||||||
|
-v verbose rsync output
|
||||||
|
|
||||||
|
data_sync rsyncs the tree at the sync source directory (-d) to the sync target
|
||||||
|
directory (-t); it checks the mount directory (-m) exists; the sync target
|
||||||
|
target directory must exist on the mount directory.
|
||||||
|
|
||||||
|
|
@ -0,0 +1,230 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.wntrmute.dev/kyle/goutils/config"
|
||||||
|
"git.wntrmute.dev/kyle/goutils/fileutil"
|
||||||
|
"git.wntrmute.dev/kyle/goutils/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
func mustHostname() string {
|
||||||
|
hostname, err := os.Hostname()
|
||||||
|
log.FatalError(err, "couldn't retrieve hostname")
|
||||||
|
|
||||||
|
if hostname == "" {
|
||||||
|
log.Fatal("no hostname returned")
|
||||||
|
}
|
||||||
|
return strings.Split(hostname, ".")[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
defaultDataDir = mustHostname() + "_data"
|
||||||
|
defaultProgName = defaultDataDir + "_sync"
|
||||||
|
defaultMountDir = filepath.Join("/media", os.Getenv("USER"), defaultDataDir)
|
||||||
|
defaultSyncDir = os.Getenv("HOME")
|
||||||
|
defaultTargetDir = filepath.Join(defaultMountDir, os.Getenv("USER"))
|
||||||
|
)
|
||||||
|
|
||||||
|
func usage(w io.Writer) {
|
||||||
|
prog := filepath.Base(os.Args[0])
|
||||||
|
fmt.Fprintf(w, `Usage: %s [-d path] [-l level] [-m path] [-nqsv]
|
||||||
|
[-t path]
|
||||||
|
-d path path to sync source directory
|
||||||
|
(default "%s")
|
||||||
|
-l level log level to output (default "INFO"). Valid log
|
||||||
|
levels are DEBUG, INFO, NOTICE, WARNING, ERR,
|
||||||
|
CRIT, ALERT, EMERG. The default is INFO.
|
||||||
|
-m path path to sync mount directory
|
||||||
|
(default "%s")
|
||||||
|
-n dry-run mode: only check paths and print files to
|
||||||
|
exclude
|
||||||
|
-q suppress console output
|
||||||
|
-s suppress syslog output
|
||||||
|
-t path path to sync target directory
|
||||||
|
(default "%s")
|
||||||
|
-v verbose rsync output
|
||||||
|
|
||||||
|
%s rsyncs the tree at the sync source directory (-d) to the sync target
|
||||||
|
directory (-t); it checks the mount directory (-m) exists; the sync target
|
||||||
|
target directory must exist on the mount directory.
|
||||||
|
|
||||||
|
`, prog, defaultSyncDir, defaultMountDir, defaultTargetDir, prog)
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkPaths(mount, target string, dryRun bool) error {
|
||||||
|
if !fileutil.DirectoryDoesExist(mount) {
|
||||||
|
return fmt.Errorf("sync dir %s isn't mounted", mount)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasPrefix(target, mount) {
|
||||||
|
return fmt.Errorf("target dir %s must exist in %s", target, mount)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !fileutil.DirectoryDoesExist(target) {
|
||||||
|
if dryRun {
|
||||||
|
log.Infof("would mkdir %s", target)
|
||||||
|
} else {
|
||||||
|
log.Infof("mkdir %s", target)
|
||||||
|
if err := os.Mkdir(target, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildExcludes(syncDir string) ([]string, error) {
|
||||||
|
var excluded []string
|
||||||
|
|
||||||
|
walker := func(path string, info fs.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
excluded = append(excluded, strings.TrimPrefix(path, syncDir))
|
||||||
|
if info != nil && info.IsDir() {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if info.Mode().IsRegular() {
|
||||||
|
if err = fileutil.Access(path, fileutil.AccessRead); err != nil {
|
||||||
|
excluded = append(excluded, strings.TrimPrefix(path, syncDir))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if info.IsDir() {
|
||||||
|
if err = fileutil.Access(path, fileutil.AccessExec); err != nil {
|
||||||
|
excluded = append(excluded, strings.TrimPrefix(path, syncDir))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
err := filepath.Walk(syncDir, walker)
|
||||||
|
return excluded, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeExcludes(excluded []string) (string, error) {
|
||||||
|
if len(excluded) == 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
excludeFile, err := os.CreateTemp("", defaultProgName)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range excluded {
|
||||||
|
fmt.Fprintln(excludeFile, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer excludeFile.Close()
|
||||||
|
return excludeFile.Name(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func rsync(syncDir, target, excludeFile string, verboseRsync bool) error {
|
||||||
|
var args []string
|
||||||
|
|
||||||
|
if excludeFile != "" {
|
||||||
|
args = append(args, "--exclude-from")
|
||||||
|
args = append(args, excludeFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
if verboseRsync {
|
||||||
|
args = append(args, "--progress")
|
||||||
|
args = append(args, "-v")
|
||||||
|
}
|
||||||
|
|
||||||
|
args = append(args, []string{"-au", syncDir + "/", target + "/"}...)
|
||||||
|
|
||||||
|
path, err := exec.LookPath("rsync")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.Command(path, args...)
|
||||||
|
cmd.Stdout = os.Stdout
|
||||||
|
cmd.Stderr = os.Stderr
|
||||||
|
return cmd.Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
flag.Usage = func() { usage(os.Stderr) }
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
|
||||||
|
var logLevel, mountDir, syncDir, target string
|
||||||
|
var dryRun, quietMode, noSyslog, verboseRsync bool
|
||||||
|
|
||||||
|
flag.StringVar(&syncDir, "d", config.GetDefault("sync_dir", defaultSyncDir),
|
||||||
|
"`path to sync source directory`")
|
||||||
|
flag.StringVar(&logLevel, "l", config.GetDefault("log_level", "INFO"),
|
||||||
|
"log level to output")
|
||||||
|
flag.StringVar(&mountDir, "m", config.GetDefault("mount_dir", defaultMountDir),
|
||||||
|
"`path` to sync mount directory")
|
||||||
|
flag.BoolVar(&dryRun, "n", false, "dry-run mode: only check paths and print files to exclude")
|
||||||
|
flag.BoolVar(&quietMode, "q", quietMode, "suppress console output")
|
||||||
|
flag.BoolVar(&noSyslog, "s", noSyslog, "suppress syslog output")
|
||||||
|
flag.StringVar(&target, "t", config.GetDefault("sync_target", defaultTargetDir),
|
||||||
|
"`path` to sync target directory")
|
||||||
|
flag.BoolVar(&verboseRsync, "v", false, "verbose rsync output")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if quietMode && noSyslog {
|
||||||
|
fmt.Fprintln(os.Stderr, "both console and syslog output are suppressed")
|
||||||
|
fmt.Fprintln(os.Stderr, "errors will NOT be reported")
|
||||||
|
}
|
||||||
|
|
||||||
|
logOpts := &log.Options{
|
||||||
|
Level: logLevel,
|
||||||
|
Tag: defaultProgName,
|
||||||
|
Facility: "user",
|
||||||
|
WriteSyslog: !noSyslog,
|
||||||
|
WriteConsole: !quietMode,
|
||||||
|
}
|
||||||
|
err := log.Setup(logOpts)
|
||||||
|
log.FatalError(err, "failed to set up logging")
|
||||||
|
|
||||||
|
log.Infof("checking paths: mount=%s, target=%s", mountDir, target)
|
||||||
|
err = checkPaths(mountDir, target, dryRun)
|
||||||
|
log.FatalError(err, "target dir isn't ready")
|
||||||
|
|
||||||
|
log.Infof("checking for files to exclude from %s", syncDir)
|
||||||
|
excluded, err := buildExcludes(syncDir)
|
||||||
|
log.FatalError(err, "couldn't build excludes")
|
||||||
|
|
||||||
|
if dryRun {
|
||||||
|
fmt.Println("excluded files:")
|
||||||
|
for _, path := range excluded {
|
||||||
|
fmt.Printf("\t%s\n", path)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
excludeFile, err := writeExcludes(excluded)
|
||||||
|
log.FatalError(err, "couldn't write exclude file")
|
||||||
|
log.Infof("excluding %d files via %s", len(excluded), excludeFile)
|
||||||
|
|
||||||
|
if excludeFile != "" {
|
||||||
|
defer func() {
|
||||||
|
log.Infof("removing exclude file %s", excludeFile)
|
||||||
|
if err := os.Remove(excludeFile); err != nil {
|
||||||
|
log.Warningf("failed to remove temp file %s", excludeFile)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
err = rsync(syncDir, target, excludeFile, verboseRsync)
|
||||||
|
log.FatalError(err, "couldn't sync data")
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue