Reuse hook mechanism for scheduling clean up tasks (#33)

* reuse hook mechanism for scheduling clean up tasks

* register hooks before creating files or dirs

* fix logging order

* use typed hook levels

README.md

@@ -126,6 +126,13 @@ You can populate below template according to your requirements and use it as you
 
 # BACKUP_LATEST_SYMLINK="backup.latest.tar.gz"
 
+# Whether to copy the content of backup folder before creating the tar archive.
+# In the rare scenario where the content of the source backup volume is continously
+# updating, but we do not wish to stop the container while performing the backup,
+# this setting can be used to ensure the integrity of the tar.gz file.
+
+# BACKUP_FROM_SNAPSHOT="false"
+
 ########### BACKUP STORAGE
 
 # The name of the remote bucket that should be used for storing backups. If

cmd/backup/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/m90/targz"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/otiai10/copy"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/openpgp"
 )
@@ -40,16 +41,31 @@ func main() {
 	}
 
 	defer func() {
-		if err := recover(); err != nil {
+		if pArg := recover(); pArg != nil {
-			if e, ok := err.(error); ok && strings.Contains(e.Error(), msgBackupFailed) {
+			if err, ok := pArg.(error); ok {
+				if hookErr := s.runHooks(err, hookLevelCleanup, hookLevelFailure); hookErr != nil {
+					s.logger.Errorf("An error occurred calling the registered hooks: %s", hookErr)
+				}
 				os.Exit(1)
 			}
-			panic(err)
+			panic(pArg)
 		}
+
+		if err := s.runHooks(nil, hookLevelCleanup); err != nil {
+			s.logger.Errorf(
+				"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
+				err,
+			)
+			os.Exit(1)
+		}
+		s.logger.Info("Finished running backup tasks.")
 	}()
 
 	s.must(func() error {
 		restartContainers, err := s.stopContainers()
+		// The mechanism for restarting containers is not using hooks as it
+		// should happen as soon as possible (i.e. before uploading backups or
+		// similar).
 		defer func() {
 			s.must(restartContainers())
 		}()
@@ -59,16 +75,9 @@ func main() {
 		return s.takeBackup()
 	}())
 
-	s.must(func() error {
+	s.must(s.encryptBackup())
-		defer func() {
+	s.must(s.copyBackup())
-			s.must(s.removeArtifacts())
-		}()
-		s.must(s.encryptBackup())
-		return s.copyBackup()
-	}())
-
 	s.must(s.pruneOldBackups())
-	s.logger.Info("Finished running backup tasks.")
 }
 
 // script holds all the stateful information required to orchestrate a
@@ -95,6 +104,7 @@ type config struct {
 	BackupPruningLeeway        time.Duration `split_words:"true" default:"1m"`
 	BackupPruningPrefix        string        `split_words:"true"`
 	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
+	BackupFromSnapshot         bool          `split_words:"true"`
 	AwsS3BucketName            string        `split_words:"true"`
 	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
 	AwsEndpointProto           string        `split_words:"true" default:"https"`
@@ -212,6 +222,11 @@ func newScript() (*script, error) {
 
 var noop = func() error { return nil }
 
+// registerHook adds the given action at the given level.
+func (s *script) registerHook(level hookLevel, action func(err error, start time.Time, logOutput string) error) {
+	s.hooks = append(s.hooks, hook{level, action})
+}
+
 // stopContainers stops all Docker containers that are marked as to being
 // stopped during the backup and returns a function that can be called to
 // restart everything that has been stopped.
@@ -330,10 +345,37 @@ func (s *script) stopContainers() (func() error, error) {
 // saves it to disk.
 func (s *script) takeBackup() error {
 	s.file = timeutil.Strftime(&s.start, s.file)
-	if err := targz.Compress(s.c.BackupSources, s.file); err != nil {
+	backupSources := s.c.BackupSources
+
+	if s.c.BackupFromSnapshot {
+		backupSources = filepath.Join("/tmp", s.c.BackupSources)
+		// copy before compressing guard against a situation where backup folder's content are still growing.
+		s.registerHook(hookLevelCleanup, func(error, time.Time, string) error {
+			if err := remove(backupSources); err != nil {
+				return fmt.Errorf("takeBackup: error removing snapshot: %w", err)
+			}
+			s.logger.Infof("Removed snapshot `%s`.", backupSources)
+			return nil
+		})
+		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{PreserveTimes: true}); err != nil {
+			return fmt.Errorf("takeBackup: error creating snapshot: %w", err)
+		}
+		s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources)
+	}
+
+	tarFile := s.file
+	s.registerHook(hookLevelCleanup, func(error, time.Time, string) error {
+		if err := remove(tarFile); err != nil {
+			return fmt.Errorf("takeBackup: error removing tar file: %w", err)
+		}
+		s.logger.Infof("Removed tar file `%s`.", tarFile)
+		return nil
+	})
+	if err := targz.Compress(backupSources, tarFile); err != nil {
 		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
 	}
-	s.logger.Infof("Created backup of `%s` at `%s`.", s.c.BackupSources, s.file)
+
+	s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, tarFile)
 	return nil
 }
 
@@ -344,9 +386,16 @@ func (s *script) encryptBackup() error {
 	if s.c.GpgPassphrase == "" {
 		return nil
 	}
-	defer os.Remove(s.file)
 
 	gpgFile := fmt.Sprintf("%s.gpg", s.file)
+	s.registerHook(hookLevelCleanup, func(error, time.Time, string) error {
+		if err := remove(gpgFile); err != nil {
+			return fmt.Errorf("encryptBackup: error removing gpg file: %w", err)
+		}
+		s.logger.Infof("Removed GPG file `%s`.", gpgFile)
+		return nil
+	})
+
 	outFile, err := os.Create(gpgFile)
 	defer outFile.Close()
 	if err != nil {
@@ -365,7 +414,7 @@ func (s *script) encryptBackup() error {
 
 	src, err := os.Open(s.file)
 	if err != nil {
-		return fmt.Errorf("encryptBackup: error opening backup file %s: %w", s.file, err)
+		return fmt.Errorf("encryptBackup: error opening backup file `%s`: %w", s.file, err)
 	}
 
 	if _, err := io.Copy(dst, src); err != nil {
@@ -392,7 +441,7 @@ func (s *script) copyBackup() error {
 	}
 
 	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
-		if err := copy(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
+		if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
 			return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
 		}
 		s.logger.Infof("Stored copy of backup `%s` in local archive `%s`.", s.file, s.c.BackupArchive)
@@ -410,22 +459,6 @@ func (s *script) copyBackup() error {
 	return nil
 }
 
-// removeArtifacts removes the backup file from disk.
-func (s *script) removeArtifacts() error {
-	_, err := os.Stat(s.file)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return fmt.Errorf("removeArtifacts: error calling stat on file %s: %w", s.file, err)
-	}
-	if err := os.Remove(s.file); err != nil {
-		return fmt.Errorf("removeArtifacts: error removing file %s: %w", s.file, err)
-	}
-	s.logger.Infof("Removed local artifacts %s.", s.file)
-	return nil
-}
-
 // pruneOldBackups rotates away backups from local and remote storages using
 // the given configuration. In case the given configuration would delete all
 // backups, it does nothing instead.
@@ -503,15 +536,35 @@ func (s *script) pruneOldBackups() error {
 	}
 
 	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
-		candidates, err := filepath.Glob(
+		globPattern := path.Join(
-			path.Join(s.c.BackupArchive, fmt.Sprintf("%s*", s.c.BackupPruningPrefix)),
+			s.c.BackupArchive,
+			fmt.Sprintf("%s*", s.c.BackupPruningPrefix),
 		)
+		globMatches, err := filepath.Glob(globPattern)
 		if err != nil {
 			return fmt.Errorf(
-				"pruneOldBackups: error looking up matching files, starting with: %w", err,
+				"pruneOldBackups: error looking up matching files using pattern %s: %w",
+				globPattern,
+				err,
 			)
 		}
 
+		var candidates []string
+		for _, candidate := range globMatches {
+			fi, err := os.Lstat(candidate)
+			if err != nil {
+				return fmt.Errorf(
+					"pruneOldBackups: error calling Lstat on file %s: %w",
+					candidate,
+					err,
+				)
+			}
+
+			if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+				candidates = append(candidates, candidate)
+			}
+		}
+
 		var matches []string
 		for _, candidate := range candidates {
 			fi, err := os.Stat(candidate)
@@ -522,16 +575,15 @@ func (s *script) pruneOldBackups() error {
 					err,
 				)
 			}
-
+			if fi.ModTime().Before(deadline) {
-			if fi.Mode() != os.ModeSymlink && fi.ModTime().Before(deadline) {
 				matches = append(matches, candidate)
 			}
 		}
 
 		if len(matches) != 0 && len(matches) != len(candidates) {
 			var removeErrors []error
-			for _, candidate := range matches {
+			for _, match := range matches {
-				if err := os.Remove(candidate); err != nil {
+				if err := os.Remove(match); err != nil {
 					removeErrors = append(removeErrors, err)
 				}
 			}
@@ -562,16 +614,18 @@ func (s *script) pruneOldBackups() error {
 }
 
 // runHooks runs all hooks that have been registered using the
-// given level. In case executing a hook returns an error, the following
+// given levels in the defined ordering. In case executing a hook returns an
-// hooks will still be run before the function returns an error.
+// error, the following hooks will still be run before the function returns.
-func (s *script) runHooks(err error, targetLevel string) error {
+func (s *script) runHooks(err error, levels ...hookLevel) error {
 	var actionErrors []error
-	for _, hook := range s.hooks {
+	for _, level := range levels {
-		if hook.level != targetLevel {
+		for _, hook := range s.hooks {
-			continue
+			if hook.level != level {
-		}
+				continue
-		if err := hook.action(err, s.start, s.output.String()); err != nil {
+			}
-			actionErrors = append(actionErrors, err)
+			if actionErr := hook.action(err, s.start, s.output.String()); actionErr != nil {
+				actionErrors = append(actionErrors, fmt.Errorf("runHooks: error running hook: %w", actionErr))
+			}
 		}
 	}
 	if len(actionErrors) != 0 {
@@ -581,18 +635,34 @@ func (s *script) runHooks(err error, targetLevel string) error {
 }
 
 // must exits the script run prematurely in case the given error
-// is non-nil. If failure hooks have been registered on the script object, they
+// is non-nil.
-// will be called, passing the failure and previous log output.
 func (s *script) must(err error) {
 	if err != nil {
 		s.logger.Errorf("Fatal error running backup: %s", err)
-		if hookErr := s.runHooks(err, hookLevelFailure); hookErr != nil {
+		panic(err)
-			s.logger.Errorf("An error occurred calling the registered failure hooks: %s", hookErr)
-		}
-		panic(errors.New(msgBackupFailed))
 	}
 }
 
+// remove removes the given file or directory from disk.
+func remove(location string) error {
+	fi, err := os.Lstat(location)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("remove: error checking for existence of `%s`: %w", location, err)
+	}
+	if fi.IsDir() {
+		err = os.RemoveAll(location)
+	} else {
+		err = os.Remove(location)
+	}
+	if err != nil {
+		return fmt.Errorf("remove: error removing `%s`: %w", location, err)
+	}
+	return nil
+}
+
 // lock opens a lockfile at the given location, keeping it locked until the
 // caller invokes the returned release func. When invoked while the file is
 // still locked the function panics.
@@ -609,7 +679,7 @@ func lock(lockfile string) func() error {
 }
 
 // copy creates a copy of the file located at `dst` at `src`.
-func copy(src, dst string) error {
+func copyFile(src, dst string) error {
 	in, err := os.Open(src)
 	if err != nil {
 		return err
@@ -666,10 +736,13 @@ func (b *bufferingWriter) Write(p []byte) (n int, err error) {
 // hook contains a queued action that can be trigger them when the script
 // reaches a certain point (e.g. unsuccessful backup)
 type hook struct {
-	level  string
+	level  hookLevel
 	action func(err error, start time.Time, logOutput string) error
 }
 
+type hookLevel int
+
 const (
-	hookLevelFailure = "failure"
+	hookLevelFailure hookLevel = iota
+	hookLevelCleanup
 )

go.mod

@@ -4,11 +4,13 @@ go 1.17
 
 require (
 	github.com/docker/docker v20.10.8+incompatible
+	github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df
 	github.com/gofrs/flock v0.8.1
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
 	github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615
 	github.com/minio/minio-go/v7 v7.0.12
+	github.com/otiai10/copy v1.6.0
 	github.com/sirupsen/logrus v1.8.1
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 )
@@ -20,7 +22,6 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.0 // indirect
 	github.com/google/uuid v1.2.0 // indirect
@@ -43,5 +44,6 @@ require (
 	google.golang.org/grpc v1.33.2 // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
+	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect
 	gopkg.in/ini.v1 v1.57.0 // indirect
 )

go.sum

@@ -494,6 +494,13 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ=
+github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
+github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -921,6 +928,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
+gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

test/cli/run.sh

@@ -23,7 +23,6 @@ docker exec minio mkdir -p /data/backup
 docker run -d \
   --name offen \
   --network test_network \
-  --label "docker-volume-backup.stop-during-backup=true" \
   -v app_data:/var/opt/offen/ \
   offen/offen:latest
 
@@ -39,6 +38,7 @@ docker run --rm \
   --env AWS_ENDPOINT_PROTO=http \
   --env AWS_S3_BUCKET_NAME=backup \
   --env BACKUP_FILENAME=test.tar.gz \
+  --env "BACKUP_FROM_SNAPSHOT=true" \
   --entrypoint backup \
   offen/docker-volume-backup:$TEST_VERSION
 
@@ -56,6 +56,6 @@ fi
 
 echo "[TEST:PASS] All containers running post backup."
 
-docker rm $(docker stop minio offen backup)
+docker rm $(docker stop minio offen)
 docker volume rm backup_data app_data
 docker network rm test_network