update github.com/otai/copy, use PreserveOwner option

* reuse hook mechanism for scheduling clean up tasks

* register hooks before creating files or dirs

* fix logging order

* use typed hook levels

Dockerfile

@@ -5,6 +5,7 @@ FROM golang:1.17-alpine as builder
 
 WORKDIR /app
 COPY go.mod go.sum ./
+RUN go mod download
 COPY cmd/backup/main.go ./cmd/backup/main.go
 RUN go build -o backup cmd/backup/main.go
 

README.md

@@ -3,17 +3,21 @@
 Backup Docker volumes locally or to any S3 compatible storage.
 
 The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup.
-It handles __recurring or one-off backups of Docker volumes__ to a __local directory__ or __any S3 compatible storage__ (or both), and __rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__.
+It handles __recurring or one-off backups of Docker volumes__ to a __local directory__ or __any S3 compatible storage__ (or both), and __rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__.
 
 <!-- MarkdownTOC -->
 
 - [Quickstart](#quickstart)
+  - [Recurring backups in a compose setup](#recurring-backups-in-a-compose-setup)
+  - [One-off backups using Docker CLI](#one-off-backups-using-docker-cli)
 - [Configuration reference](#configuration-reference)
 - [How to](#how-to)
   - [Stopping containers during backup](#stopping-containers-during-backup)
   - [Automatically pruning old backups](#automatically-pruning-old-backups)
+  - [Send email notifications on failed backup runs](#send-email-notifications-on-failed-backup-runs)
   - [Encrypting your backup using GPG](#encrypting-your-backup-using-gpg)
   - [Restoring a volume from a backup](#restoring-a-volume-from-a-backup)
+  - [Set the timezone the container runs in](#set-the-timezone-the-container-runs-in)
   - [Using with Docker Swarm](#using-with-docker-swarm)
   - [Manually triggering a backup](#manually-triggering-a-backup)
 - [Recipes](#recipes)
@@ -37,6 +41,8 @@ Code and documentation for `v1` versions are found on [this branch][v1-branch].
 
 ## Quickstart
 
+### Recurring backups in a compose setup
+
 Add a `backup` service to your compose setup and mount the volumes you would like to see backed up:
 
 ```yml
@@ -55,6 +61,10 @@ services:
       - docker-volume-backup.stop-during-backup=true
 
   backup:
+    # In production, it is advised to lock your image tag to a proper
+    # release version instead of using `latest`.
+    # Check https://github.com/offen/docker-volume-backup/releases
+    # for a list of available releases.
     image: offen/docker-volume-backup:latest
     restart: always
     env_file: ./backup.env # see below for configuration reference
@@ -73,6 +83,22 @@ volumes:
   data:
 ```
 
+### One-off backups using Docker CLI
+
+To run a one time backup, mount the volume you would like to see backed up into a container and run the `backup` command:
+
+```console
+docker run --rm \
+  -v data:/backup/data \
+  --env AWS_ACCESS_KEY_ID="<xxx>" \
+  --env AWS_SECRET_ACCESS_KEY="<xxx>" \
+  --env AWS_S3_BUCKET_NAME="<xxx>" \
+  --entrypoint backup \
+  offen/docker-volume-backup:latest
+```
+
+Alternatively, pass a `--env-file` in order to use a full config as described below.
+
 ## Configuration reference
 
 Backup targets, schedule and retention are configured in environment variables.
@@ -95,6 +121,18 @@ You can populate below template according to your requirements and use it as you
 
 # BACKUP_FILENAME="backup-%Y-%m-%dT%H-%M-%S.tar.gz"
 
+# When storing local backups, a symlink to the latest backup can be created
+# in case a value is given for this key. This has no effect on remote backups.
+
+# BACKUP_LATEST_SYMLINK="backup.latest.tar.gz"
+
+# Whether to copy the content of backup folder before creating the tar archive.
+# In the rare scenario where the content of the source backup volume is continously
+# updating, but we do not wish to stop the container while performing the backup,
+# this setting can be used to ensure the integrity of the tar.gz file.
+
+# BACKUP_FROM_SNAPSHOT="false"
+
 ########### BACKUP STORAGE
 
 # The name of the remote bucket that should be used for storing backups. If
@@ -109,6 +147,13 @@ You can populate below template according to your requirements and use it as you
 # AWS_ACCESS_KEY_ID="<xxx>"
 # AWS_SECRET_ACCESS_KEY="<xxx>"
 
+# Instead of providing static credentials, you can also use IAM instance profiles
+# or similar to provide authentication. Some possible configuration options on AWS:
+# - EC2: http://169.254.169.254
+# - ECS: http://169.254.170.2
+
+# AWS_IAM_ROLE_ENDPOINT="http://169.254.169.254"
+
 # This is the FQDN of your storage server, e.g. `storage.example.com`.
 # Do not set this when working against AWS S3 (the default value is 
 # `s3.amazonaws.com`). If you need to set a specific (non-https) protocol, you
@@ -124,7 +169,8 @@ You can populate below template according to your requirements and use it as you
 
 # Setting this variable to `true` will disable verification of
 # SSL certificates. You shouldn't use this unless you use self-signed
-# certificates for your remote storage backend.
+# certificates for your remote storage backend. This can only be used
+# when AWS_ENDPOINT_PROTO is set to `https`.
 
 # AWS_ENDPOINT_INSECURE="true"
 
@@ -188,6 +234,30 @@ You can populate below template according to your requirements and use it as you
 # override this default by specifying a different value here.
 
 # BACKUP_STOP_CONTAINER_LABEL="service1"
+
+########### EMAIL NOTIFICATIONS ON FAILED BACKUP RUNS
+
+# In case SMTP credentials are provided, notification emails can be sent out on
+# failed backup runs. These emails will contain the start time, the error
+# message and all log output prior to the failure.
+
+# The recipient(s) of the notification. Supply a comma separated list
+# of adresses if you want to notify multiple recipients. If this is
+# not set, no emails will be sent.
+
+# EMAIL_NOTIFICATION_RECIPIENT="you@example.com"
+
+# The "From" header of the sent email. Defaults to `noreply@nohost`.
+
+# EMAIL_NOTIFICATION_SENDER="no-reply@example.com"
+
+# Configuration and credentials for the SMTP server to be used.
+# EMAIL_SMTP_PORT defaults to 587.
+
+# EMAIL_SMTP_HOST="posteo.de"
+# EMAIL_SMTP_PASSWORD="<xxx>"
+# EMAIL_SMTP_USERNAME="no-reply@example.com"
+# EMAIL_SMTP_PORT="<port>"
 ```
 
 ## How to
@@ -247,6 +317,25 @@ volumes:
   data:
 ```
 
+### Send email notifications on failed backup runs
+
+To send out email notifications on failed backup runs, provide SMTP credentials, a sender and a recipient:
+
+```yml
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:latest
+    environment:
+      # ... other configuration values go here
+      EMAIL_SMTP_HOST: "smtp.example.com"
+      EMAIL_SMTP_PASSWORD: "password"
+      EMAIL_SMTP_USERNAME: "username"
+      EMAIL_NOTIFICATION_SENDER: "noreply@example.com"
+      EMAIL_NOTIFICATION_RECIPIENT: "notifications@example.com"
+```
+
 ### Encrypting your backup using GPG
 
 The image supports encrypting backups using GPG out of the box.
@@ -278,6 +367,27 @@ In case you need to restore a volume from a backup, the most straight forward pr
 
 Depending on your setup and the application(s) you are running, this might involve other steps to be taken still.
 
+### Set the timezone the container runs in
+
+By default a container based on this image will run in the UTC timezone.
+As the image is designed to be as small as possible, additional timezone data is not included.
+In case you want to run your cron rules in your local timezone (respecting DST and similar), you can mount your Docker host's `/etc/timezone` and `/etc/localtime` in read-only mode:
+
+```yml
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:latest
+    volumes:
+      - data:/backup/my-app-backup:ro
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+
+volumes:
+  data:
+```
+
 ### Using with Docker Swarm
 
 By default, Docker Swarm will restart stopped containers automatically, even when manually stopped.
@@ -362,6 +472,9 @@ services:
   # ... define other services using the `data` volume here
   backup:
     image: offen/docker-volume-backup:latest
+    environment:
+      BACKUP_FILENAME: backup-%Y-%m-%dT%H-%M-%S.tar.gz
+      BACKUP_LATEST_SYMLINK: backup-latest.tar.gz
     volumes:
       - data:/backup/my-app-backup:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
@@ -499,7 +612,7 @@ volumes:
 
 ## Differences to `futurice/docker-volume-backup`
 
-This image is heavily inspired by the `futurice/docker-volume-backup`. We decided to publish this image as a simpler and more lightweight alternative because of the following requirements:
+This image is heavily inspired by `futurice/docker-volume-backup`. We decided to publish this image as a simpler and more lightweight alternative because of the following requirements:
 
 - The original image is based on `ubuntu` and requires additional tools, making it heavy.
 This version is roughly 1/25 in compressed size (it's ~12MB).
@@ -510,3 +623,5 @@ Local copies of backups can also be pruned once they reach a certain age.
 - InfluxDB specific functionality from the original image was removed.
 - `arm64` and `arm/v7` architectures are supported.
 - Docker in Swarm mode is supported.
+- Notifications on failed backups are supported
+- IAM authentication through instance profiles is supported

cmd/backup/main.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -18,12 +19,14 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/client"
+	"github.com/go-gomail/gomail"
 	"github.com/gofrs/flock"
 	"github.com/kelseyhightower/envconfig"
 	"github.com/leekchan/timeutil"
 	"github.com/m90/targz"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/otiai10/copy"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/openpgp"
 )
@@ -37,8 +40,32 @@ func main() {
 		panic(err)
 	}
 
+	defer func() {
+		if pArg := recover(); pArg != nil {
+			if err, ok := pArg.(error); ok {
+				if hookErr := s.runHooks(err, hookLevelCleanup, hookLevelFailure); hookErr != nil {
+					s.logger.Errorf("An error occurred calling the registered hooks: %s", hookErr)
+				}
+				os.Exit(1)
+			}
+			panic(pArg)
+		}
+
+		if err := s.runHooks(nil, hookLevelCleanup); err != nil {
+			s.logger.Errorf(
+				"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
+				err,
+			)
+			os.Exit(1)
+		}
+		s.logger.Info("Finished running backup tasks.")
+	}()
+
 	s.must(func() error {
 		restartContainers, err := s.stopContainers()
+		// The mechanism for restarting containers is not using hooks as it
+		// should happen as soon as possible (i.e. before uploading backups or
+		// similar).
 		defer func() {
 			s.must(restartContainers())
 		}()
@@ -50,9 +77,7 @@ func main() {
 
 	s.must(s.encryptBackup())
 	s.must(s.copyBackup())
-	s.must(s.removeArtifacts())
 	s.must(s.pruneOldBackups())
-	s.logger.Info("Finished running backup tasks.")
 }
 
 // script holds all the stateful information required to orchestrate a
@@ -61,9 +86,11 @@ type script struct {
 	cli    *client.Client
 	mc     *minio.Client
 	logger *logrus.Logger
+	hooks  []hook
 
 	start  time.Time
 	file   string
+	output *bytes.Buffer
 
 	c *config
 }
@@ -71,34 +98,47 @@ type script struct {
 type config struct {
 	BackupSources              string        `split_words:"true" default:"/backup"`
 	BackupFilename             string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
+	BackupLatestSymlink        string        `split_words:"true"`
 	BackupArchive              string        `split_words:"true" default:"/archive"`
 	BackupRetentionDays        int32         `split_words:"true" default:"-1"`
 	BackupPruningLeeway        time.Duration `split_words:"true" default:"1m"`
 	BackupPruningPrefix        string        `split_words:"true"`
 	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
+	BackupFromSnapshot         bool          `split_words:"true"`
 	AwsS3BucketName            string        `split_words:"true"`
 	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
 	AwsEndpointProto           string        `split_words:"true" default:"https"`
 	AwsEndpointInsecure        bool          `split_words:"true"`
 	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
 	AwsSecretAccessKey         string        `split_words:"true"`
+	AwsIamRoleEndpoint         string        `split_words:"true"`
 	GpgPassphrase              string        `split_words:"true"`
+	EmailNotificationRecipient string        `split_words:"true"`
+	EmailNotificationSender    string        `split_words:"true" default:"noreply@nohost"`
+	EmailSMTPHost              string        `envconfig:"EMAIL_SMTP_HOST"`
+	EmailSMTPPort              int           `envconfig:"EMAIL_SMTP_PORT" default:"587"`
+	EmailSMTPUsername          string        `envconfig:"EMAIL_SMTP_USERNAME"`
+	EmailSMTPPassword          string        `envconfig:"EMAIL_SMTP_PASSWORD"`
 }
 
+var msgBackupFailed = "backup run failed"
+
 // newScript creates all resources needed for the script to perform actions against
 // remote resources like the Docker engine or remote storage locations. All
 // reading from env vars or other configuration sources is expected to happen
 // in this method.
 func newScript() (*script, error) {
+	stdOut, logBuffer := buffer(os.Stdout)
 	s := &script{
 		c: &config{},
 		logger: &logrus.Logger{
-			Out:       os.Stdout,
+			Out:       stdOut,
 			Formatter: new(logrus.TextFormatter),
 			Hooks:     make(logrus.LevelHooks),
 			Level:     logrus.InfoLevel,
 		},
 		start:  time.Now(),
+		output: logBuffer,
 	}
 
 	if err := envconfig.Process("", s.c); err != nil {
@@ -117,25 +157,76 @@ func newScript() (*script, error) {
 	}
 
 	if s.c.AwsS3BucketName != "" {
-		mc, err := minio.New(s.c.AwsEndpoint, &minio.Options{
-			Creds: credentials.NewStaticV4(
+		var creds *credentials.Credentials
+		if s.c.AwsAccessKeyID != "" && s.c.AwsSecretAccessKey != "" {
+			creds = credentials.NewStaticV4(
 				s.c.AwsAccessKeyID,
 				s.c.AwsSecretAccessKey,
 				"",
-			),
-			Secure: !s.c.AwsEndpointInsecure && s.c.AwsEndpointProto == "https",
-		})
+			)
+		} else if s.c.AwsIamRoleEndpoint != "" {
+			creds = credentials.NewIAM(s.c.AwsIamRoleEndpoint)
+		} else {
+			return nil, errors.New("newScript: AWS_S3_BUCKET_NAME is defined, but no credentials were provided")
+		}
+
+		options := minio.Options{
+			Creds:  creds,
+			Secure: s.c.AwsEndpointProto == "https",
+		}
+
+		if s.c.AwsEndpointInsecure {
+			if !options.Secure {
+				return nil, errors.New("newScript: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
+			}
+
+			transport, err := minio.DefaultTransport(true)
+			if err != nil {
+				return nil, fmt.Errorf("newScript: failed to create default minio transport")
+			}
+			transport.TLSClientConfig.InsecureSkipVerify = true
+			options.Transport = transport
+		}
+
+		mc, err := minio.New(s.c.AwsEndpoint, &options)
 		if err != nil {
 			return nil, fmt.Errorf("newScript: error setting up minio client: %w", err)
 		}
 		s.mc = mc
 	}
 
+	if s.c.EmailNotificationRecipient != "" {
+		s.hooks = append(s.hooks, hook{hookLevelFailure, func(err error, start time.Time, logOutput string) error {
+			mailer := gomail.NewDialer(
+				s.c.EmailSMTPHost, s.c.EmailSMTPPort, s.c.EmailSMTPUsername, s.c.EmailSMTPPassword,
+			)
+
+			subject := fmt.Sprintf(
+				"Failure running docker-volume-backup at %s", start.Format(time.RFC3339),
+			)
+			body := fmt.Sprintf(
+				"Running docker-volume-backup failed with error: %s\n\nLog output of the failed run was:\n\n%s\n", err, logOutput,
+			)
+
+			message := gomail.NewMessage()
+			message.SetHeader("From", s.c.EmailNotificationSender)
+			message.SetHeader("To", s.c.EmailNotificationRecipient)
+			message.SetHeader("Subject", subject)
+			message.SetBody("text/plain", body)
+			return mailer.DialAndSend(message)
+		}})
+	}
+
 	return s, nil
 }
 
 var noop = func() error { return nil }
 
+// registerHook adds the given action at the given level.
+func (s *script) registerHook(level hookLevel, action func(err error, start time.Time, logOutput string) error) {
+	s.hooks = append(s.hooks, hook{level, action})
+}
+
 // stopContainers stops all Docker containers that are marked as to being
 // stopped during the backup and returns a function that can be called to
 // restart everything that has been stopped.
@@ -225,11 +316,10 @@ func (s *script) stopContainers() (func() error, error) {
 					return fmt.Errorf("stopContainersAndRun: couldn't find service with name %s", serviceName)
 				}
 				serviceMatch.Spec.TaskTemplate.ForceUpdate = 1
-				_, err := s.cli.ServiceUpdate(
+				if _, err := s.cli.ServiceUpdate(
 					context.Background(), serviceMatch.ID,
 					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},
-				)
-				if err != nil {
+				); err != nil {
 					restartErrors = append(restartErrors, err)
 				}
 			}
@@ -254,10 +344,40 @@ func (s *script) stopContainers() (func() error, error) {
 // saves it to disk.
 func (s *script) takeBackup() error {
 	s.file = timeutil.Strftime(&s.start, s.file)
-	if err := targz.Compress(s.c.BackupSources, s.file); err != nil {
+	backupSources := s.c.BackupSources
+
+	if s.c.BackupFromSnapshot {
+		backupSources = filepath.Join("/tmp", s.c.BackupSources)
+		// copy before compressing guard against a situation where backup folder's content are still growing.
+		s.registerHook(hookLevelCleanup, func(error, time.Time, string) error {
+			if err := remove(backupSources); err != nil {
+				return fmt.Errorf("takeBackup: error removing snapshot: %w", err)
+			}
+			s.logger.Infof("Removed snapshot `%s`.", backupSources)
+			return nil
+		})
+		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
+			PreserveTimes: true,
+			PreserveOwner: true,
+		}); err != nil {
+			return fmt.Errorf("takeBackup: error creating snapshot: %w", err)
+		}
+		s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources)
+	}
+
+	tarFile := s.file
+	s.registerHook(hookLevelCleanup, func(error, time.Time, string) error {
+		if err := remove(tarFile); err != nil {
+			return fmt.Errorf("takeBackup: error removing tar file: %w", err)
+		}
+		s.logger.Infof("Removed tar file `%s`.", tarFile)
+		return nil
+	})
+	if err := targz.Compress(backupSources, tarFile); err != nil {
 		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
 	}
-	s.logger.Infof("Created backup of `%s` at `%s`.", s.c.BackupSources, s.file)
+
+	s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, tarFile)
 	return nil
 }
 
@@ -268,9 +388,16 @@ func (s *script) encryptBackup() error {
 	if s.c.GpgPassphrase == "" {
 		return nil
 	}
-	defer os.Remove(s.file)
 
 	gpgFile := fmt.Sprintf("%s.gpg", s.file)
+	s.registerHook(hookLevelCleanup, func(error, time.Time, string) error {
+		if err := remove(gpgFile); err != nil {
+			return fmt.Errorf("encryptBackup: error removing gpg file: %w", err)
+		}
+		s.logger.Infof("Removed GPG file `%s`.", gpgFile)
+		return nil
+	})
+
 	outFile, err := os.Create(gpgFile)
 	defer outFile.Close()
 	if err != nil {
@@ -289,7 +416,7 @@ func (s *script) encryptBackup() error {
 
 	src, err := os.Open(s.file)
 	if err != nil {
-		return fmt.Errorf("encryptBackup: error opening backup file %s: %w", s.file, err)
+		return fmt.Errorf("encryptBackup: error opening backup file `%s`: %w", s.file, err)
 	}
 
 	if _, err := io.Copy(dst, src); err != nil {
@@ -306,30 +433,30 @@ func (s *script) encryptBackup() error {
 func (s *script) copyBackup() error {
 	_, name := path.Split(s.file)
 	if s.mc != nil {
-		_, err := s.mc.FPutObject(context.Background(), s.c.AwsS3BucketName, name, s.file, minio.PutObjectOptions{
+		if _, err := s.mc.FPutObject(context.Background(), s.c.AwsS3BucketName, name, s.file, minio.PutObjectOptions{
 			ContentType: "application/tar+gzip",
-		})
-		if err != nil {
+		}); err != nil {
 			return fmt.Errorf("copyBackup: error uploading backup to remote storage: %w", err)
 		}
 		s.logger.Infof("Uploaded a copy of backup `%s` to bucket `%s`.", s.file, s.c.AwsS3BucketName)
 	}
 
 	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
-		if err := copy(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
+		if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
 			return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
 		}
 		s.logger.Infof("Stored copy of backup `%s` in local archive `%s`.", s.file, s.c.BackupArchive)
+		if s.c.BackupLatestSymlink != "" {
+			symlink := path.Join(s.c.BackupArchive, s.c.BackupLatestSymlink)
+			if _, err := os.Lstat(symlink); err == nil {
+				os.Remove(symlink)
 			}
-	return nil
+			if err := os.Symlink(name, symlink); err != nil {
+				return fmt.Errorf("copyBackup: error creating latest symlink: %w", err)
+			}
+			s.logger.Infof("Created/Updated symlink `%s` for latest backup.", s.c.BackupLatestSymlink)
 		}
-
-// removeArtifacts removes the backup file from disk.
-func (s *script) removeArtifacts() error {
-	if err := os.Remove(s.file); err != nil {
-		return fmt.Errorf("removeArtifacts: error removing file: %w", err)
 	}
-	s.logger.Info("Removed local artifacts.")
 	return nil
 }
 
@@ -410,15 +537,35 @@ func (s *script) pruneOldBackups() error {
 	}
 
 	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
-		candidates, err := filepath.Glob(
-			path.Join(s.c.BackupArchive, fmt.Sprintf("%s*", s.c.BackupPruningPrefix)),
+		globPattern := path.Join(
+			s.c.BackupArchive,
+			fmt.Sprintf("%s*", s.c.BackupPruningPrefix),
 		)
+		globMatches, err := filepath.Glob(globPattern)
 		if err != nil {
 			return fmt.Errorf(
-				"pruneOldBackups: error looking up matching files, starting with: %w", err,
+				"pruneOldBackups: error looking up matching files using pattern %s: %w",
+				globPattern,
+				err,
 			)
 		}
 
+		var candidates []string
+		for _, candidate := range globMatches {
+			fi, err := os.Lstat(candidate)
+			if err != nil {
+				return fmt.Errorf(
+					"pruneOldBackups: error calling Lstat on file %s: %w",
+					candidate,
+					err,
+				)
+			}
+
+			if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+				candidates = append(candidates, candidate)
+			}
+		}
+
 		var matches []string
 		for _, candidate := range candidates {
 			fi, err := os.Stat(candidate)
@@ -429,7 +576,6 @@ func (s *script) pruneOldBackups() error {
 					err,
 				)
 			}
-
 			if fi.ModTime().Before(deadline) {
 				matches = append(matches, candidate)
 			}
@@ -437,8 +583,8 @@ func (s *script) pruneOldBackups() error {
 
 		if len(matches) != 0 && len(matches) != len(candidates) {
 			var removeErrors []error
-			for _, candidate := range matches {
-				if err := os.Remove(candidate); err != nil {
+			for _, match := range matches {
+				if err := os.Remove(match); err != nil {
 					removeErrors = append(removeErrors, err)
 				}
 			}
@@ -468,14 +614,56 @@ func (s *script) pruneOldBackups() error {
 	return nil
 }
 
-// must exits the script run non-zero and prematurely in case the given error
+// runHooks runs all hooks that have been registered using the
+// given levels in the defined ordering. In case executing a hook returns an
+// error, the following hooks will still be run before the function returns.
+func (s *script) runHooks(err error, levels ...hookLevel) error {
+	var actionErrors []error
+	for _, level := range levels {
+		for _, hook := range s.hooks {
+			if hook.level != level {
+				continue
+			}
+			if actionErr := hook.action(err, s.start, s.output.String()); actionErr != nil {
+				actionErrors = append(actionErrors, fmt.Errorf("runHooks: error running hook: %w", actionErr))
+			}
+		}
+	}
+	if len(actionErrors) != 0 {
+		return join(actionErrors...)
+	}
+	return nil
+}
+
+// must exits the script run prematurely in case the given error
 // is non-nil.
 func (s *script) must(err error) {
 	if err != nil {
-		s.logger.Fatalf("Fatal error running backup: %s", err)
+		s.logger.Errorf("Fatal error running backup: %s", err)
+		panic(err)
 	}
 }
 
+// remove removes the given file or directory from disk.
+func remove(location string) error {
+	fi, err := os.Lstat(location)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("remove: error checking for existence of `%s`: %w", location, err)
+	}
+	if fi.IsDir() {
+		err = os.RemoveAll(location)
+	} else {
+		err = os.Remove(location)
+	}
+	if err != nil {
+		return fmt.Errorf("remove: error removing `%s`: %w", location, err)
+	}
+	return nil
+}
+
 // lock opens a lockfile at the given location, keeping it locked until the
 // caller invokes the returned release func. When invoked while the file is
 // still locked the function panics.
@@ -492,7 +680,7 @@ func lock(lockfile string) func() error {
 }
 
 // copy creates a copy of the file located at `dst` at `src`.
-func copy(src, dst string) error {
+func copyFile(src, dst string) error {
 	in, err := os.Open(src)
 	if err != nil {
 		return err
@@ -526,3 +714,36 @@ func join(errs ...error) error {
 	}
 	return errors.New("[" + strings.Join(msgs, ", ") + "]")
 }
+
+// buffer takes an io.Writer and returns a wrapped version of the
+// writer that writes to both the original target as well as the returned buffer
+func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {
+	buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w}
+	return buffering, &buffering.buf
+}
+
+type bufferingWriter struct {
+	buf    bytes.Buffer
+	writer io.Writer
+}
+
+func (b *bufferingWriter) Write(p []byte) (n int, err error) {
+	if n, err := b.buf.Write(p); err != nil {
+		return n, fmt.Errorf("bufferingWriter: error writing to buffer: %w", err)
+	}
+	return b.writer.Write(p)
+}
+
+// hook contains a queued action that can be trigger them when the script
+// reaches a certain point (e.g. unsuccessful backup)
+type hook struct {
+	level  hookLevel
+	action func(err error, start time.Time, logOutput string) error
+}
+
+type hookLevel int
+
+const (
+	hookLevelFailure hookLevel = iota
+	hookLevelCleanup
+)

go.mod

@@ -3,12 +3,14 @@ module github.com/offen/docker-volume-backup
 go 1.17
 
 require (
-	github.com/docker/docker v20.10.8+incompatible
+	github.com/docker/docker v20.10.11+incompatible
+	github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df
 	github.com/gofrs/flock v0.8.1
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
 	github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615
-	github.com/minio/minio-go/v7 v7.0.12
+	github.com/minio/minio-go/v7 v7.0.16
+	github.com/otiai10/copy v1.7.0
 	github.com/sirupsen/logrus v1.8.1
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 )
@@ -22,25 +24,27 @@ require (
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.0 // indirect
-	github.com/google/uuid v1.2.0 // indirect
-	github.com/json-iterator/go v1.1.10 // indirect
-	github.com/klauspost/cpuid v1.3.1 // indirect
-	github.com/minio/md5-simd v1.1.0 // indirect
-	github.com/minio/sha256-simd v0.1.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rs/xid v1.2.1 // indirect
-	github.com/walle/targz v0.0.0-20140417120357-57fe4206da5a // indirect
+	github.com/rs/xid v1.3.0 // indirect
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
 	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
-	golang.org/x/text v0.3.4 // indirect
+	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
 	google.golang.org/grpc v1.33.2 // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
-	gopkg.in/ini.v1 v1.57.0 // indirect
+	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
+	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect
+	gopkg.in/ini.v1 v1.65.0 // indirect
 )

go.sum

@@ -218,8 +218,8 @@ github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TT
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.8+incompatible h1:RVqD337BgQicVCzYrrlhLDWhq6OAD2PJDUg2LsEUvKM=
-github.com/docker/docker v20.10.8+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo=
+github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
@@ -254,6 +254,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E=
+github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y=
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -336,12 +338,12 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I=
@@ -375,11 +377,11 @@ github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
@@ -390,9 +392,16 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -407,12 +416,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs=
 github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4=
-github.com/m90/targz v0.0.0-20140417120357-57fe4206da5a h1:ML3GxKU275DDCbdDvqoA2yy5kF/xOzOOmUdY4uQryUM=
-github.com/m90/targz v0.0.0-20140417120357-57fe4206da5a/go.mod h1:bWDufAy+5ZjKRbxe8gh/oKrGC+Ryww8AKPsFiwPuAGY=
-github.com/m90/targz v0.0.0-20210903165208-2619a50ded33 h1:YFpJF7QUAb2z/pviBm58vIl5VS+qD3ibzvlhqeL9TTo=
-github.com/m90/targz v0.0.0-20210903165208-2619a50ded33/go.mod h1:YZK3bSO/oVlk9G+v00BxgzxW2Us4p/R4ysHOBjk0fJI=
-github.com/m90/targz v0.0.0-20210904080153-f3929d2235bc h1:cNfcJUU7pra4Kz6XN1GbF6bHg7KWDitCxn1z4rJIqc8=
-github.com/m90/targz v0.0.0-20210904080153-f3929d2235bc/go.mod h1:YZK3bSO/oVlk9G+v00BxgzxW2Us4p/R4ysHOBjk0fJI=
 github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615 h1:rn0LO2tQEgCDOct8qnbcslTUpAIWdVlWcGkjoumhf2U=
 github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615/go.mod h1:YZK3bSO/oVlk9G+v00BxgzxW2Us4p/R4ysHOBjk0fJI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -427,12 +430,14 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
-github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
-github.com/minio/minio-go/v7 v7.0.12 h1:/4pxUdwn9w0QEryNkrrWaodIESPRX+NxpO0Q6hVdaAA=
-github.com/minio/minio-go/v7 v7.0.12/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
-github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4=
+github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -448,8 +453,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
@@ -498,6 +504,13 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
+github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI=
+github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -540,8 +553,9 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -556,10 +570,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -604,8 +616,6 @@ github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:tw
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/walle/targz v0.0.0-20140417120357-57fe4206da5a h1:6cKSHLRphD9Fo1LJlISiulvgYCIafJ3QfKLimPYcAGc=
-github.com/walle/targz v0.0.0-20140417120357-57fe4206da5a/go.mod h1:nccQrXCnc5SjsThFLmL7hYbtT/mHJcuolPifzY5vJqE=
 github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -796,8 +806,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -916,6 +927,8 @@ google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/l
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
+gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -925,9 +938,12 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
+gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.65.0 h1:B2//IEITFk89S+Nl2tozBeqUvFEpUAY6daarSlrx8jU=
+gopkg.in/ini.v1 v1.65.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=

test/cli/run.sh

@@ -23,14 +23,12 @@ docker exec minio mkdir -p /data/backup
 docker run -d \
   --name offen \
   --network test_network \
-  --label "docker-volume-backup.stop-during-backup=true" \
   -v app_data:/var/opt/offen/ \
   offen/offen:latest
 
 sleep 10
 
-docker run -d \
-  --name backup \
+docker run --rm \
   --network test_network \
   -v app_data:/backup/app_data \
   -v /var/run/docker.sock:/var/run/docker.sock \
@@ -40,18 +38,17 @@ docker run -d \
   --env AWS_ENDPOINT_PROTO=http \
   --env AWS_S3_BUCKET_NAME=backup \
   --env BACKUP_FILENAME=test.tar.gz \
-  --env BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?" \
+  --env "BACKUP_FROM_SNAPSHOT=true" \
+  --entrypoint backup \
   offen/docker-volume-backup:$TEST_VERSION
 
-docker exec backup backup
-
 docker run --rm -it \
   -v backup_data:/data alpine \
   ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
 
 echo "[TEST:PASS] Found relevant files in untared backup."
 
-if [ "$(docker ps -q | wc -l)" != "3" ]; then
+if [ "$(docker ps -q | wc -l)" != "2" ]; then
   echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
   docker ps
   exit 1
@@ -59,6 +56,6 @@ fi
 
 echo "[TEST:PASS] All containers running post backup."
 
-docker rm $(docker stop minio offen backup)
+docker rm $(docker stop minio offen)
 docker volume rm backup_data app_data
 docker network rm test_network

test/compose/docker-compose.yml

@@ -24,6 +24,7 @@ services:
       AWS_ENDPOINT_PROTO: http
       AWS_S3_BUCKET_NAME: backup
       BACKUP_FILENAME: test.tar.gz
+      BACKUP_LATEST_SYMLINK: test.latest.tar.gz.gpg
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
       BACKUP_PRUNING_LEEWAY: 5s

test/compose/run.sh

@@ -18,6 +18,7 @@ docker run --rm -it \
 
 echo "[TEST:PASS] Found relevant files in untared remote backup."
 
+test -L ./local/test.latest.tar.gz.gpg
 echo 1234secret | gpg -d --yes --passphrase-fd 0 ./local/test.tar.gz.gpg > ./local/decrypted.tar.gz
 tar -xf ./local/decrypted.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
 rm ./local/decrypted.tar.gz