Update webdav client library (#97)

The README was using `AWS_BUCKET_NAME` instead of `AWS_S3_BUCKET_NAME` in the recipes. 
This resulted in no data being uploaded to S3.

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 jobs:
   canary:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-2004:202201-02
     working_directory: ~/docker-volume-backup
     steps:
       - checkout
@@ -11,10 +11,15 @@ jobs:
           name: Build
           command: |
             docker build . -t offen/docker-volume-backup:canary
+      - run:
+          name: Install gnupg
+          command: |
+            sudo apt-get install -y gnupg
       - run:
           name: Run tests
           working_directory: ~/docker-volume-backup/test
           command: |
+            export GPG_TTY=$(tty)
             ./test.sh canary
 
   build:
@@ -39,10 +44,14 @@ jobs:
             docker context create docker-volume-backup
             docker buildx create docker-volume-backup --name docker-volume-backup --use
             docker buildx inspect --bootstrap
+            tag_args="-t offen/docker-volume-backup:$CIRCLE_TAG"
+            if [[ "$CIRCLE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+              # prerelease tags like `v2.0.0-alpha.1` should not be released as `latest`
+              tag_args="$tag_args -t offen/docker-volume-backup:latest"
+              tag_args="$tag_args -t offen/docker-volume-backup:$(echo "$CIRCLE_TAG" | cut -d. -f1)"
+            fi
             docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 \
-              -t offen/docker-volume-backup:$CIRCLE_TAG \
-              -t offen/docker-volume-backup:latest \
-              . --push
+               $tag_args . --push
 
 workflows:
   version: 2

.editorconfig

@@ -0,0 +1,18 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.go]
+indent_style = tab

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,20 @@
+* **I'm submitting a ...**
+  - [ ] bug report
+  - [ ] feature request
+  - [ ] support request
+
+* **What is the current behavior?**
+
+* **If the current behavior is a bug, please provide the configuration and steps to reproduce and if possible a minimal demo of the problem.**
+
+* **What is the expected behavior?**
+
+* **What is the motivation / use case for changing the behavior?**
+
+* **Please tell us about your environment:**
+  
+  - Image version:
+  - Docker version: 
+  - docker-compose version:
+
+* **Other information** (e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, eg. stackoverflow, etc)

Dockerfile

@@ -1,23 +1,24 @@
 # Copyright 2021 - Offen Authors <hioffen@posteo.de>
 # SPDX-License-Identifier: MPL-2.0
 
-FROM golang:1.17-alpine as builder
-ARG MC_VERSION=RELEASE.2021-06-13T17-48-22Z
-RUN go install -ldflags "-X github.com/minio/mc/cmd.ReleaseTag=$MC_VERSION" github.com/minio/mc@$MC_VERSION
+FROM golang:1.18-alpine as builder
 
-FROM alpine:3.14
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY cmd/backup ./cmd/backup/
+WORKDIR /app/cmd/backup
+RUN go build -o backup .
+
+FROM alpine:3.15
 
 WORKDIR /root
 
-RUN apk add --update ca-certificates docker openrc gnupg
-RUN update-ca-certificates
-RUN rc-update add docker boot
+RUN apk add --no-cache ca-certificates
 
-COPY --from=builder /go/bin/mc /usr/bin/mc
-RUN mc --version
+COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
 
-COPY src/backup.sh src/entrypoint.sh /root/
-RUN chmod +x backup.sh && mv backup.sh /usr/bin/backup \
-  && chmod +x entrypoint.sh
+COPY ./entrypoint.sh /root/
+RUN chmod +x entrypoint.sh
 
 ENTRYPOINT ["/root/entrypoint.sh"]

NOTICE

@@ -1,22 +0,0 @@
-Copyright 2021 Offen Authors <hioffen@posteo.de>
-
-This project contains software that is Copyright (c) 2018 Futurice
-Licensed under the Terms of the MIT License:
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/backup/archive.go

@@ -0,0 +1,142 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+// Portions of this file are taken from package `targz`, Copyright (c) 2014 Fredrik Wallgren
+// Licensed under the MIT License: https://github.com/walle/targz/blob/57fe4206da5abf7dd3901b4af3891ec2f08c7b08/LICENSE
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+func createArchive(inputFilePath, outputFilePath string) error {
+	inputFilePath = stripTrailingSlashes(inputFilePath)
+	inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
+	if err != nil {
+		return fmt.Errorf("createArchive: error transposing given file paths: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(outputFilePath), 0755); err != nil {
+		return fmt.Errorf("createArchive: error creating output file path: %w", err)
+	}
+
+	if err := compress(inputFilePath, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
+		return fmt.Errorf("createArchive: error creating archive: %w", err)
+	}
+
+	return nil
+}
+
+func stripTrailingSlashes(path string) string {
+	if len(path) > 0 && path[len(path)-1] == '/' {
+		path = path[0 : len(path)-1]
+	}
+
+	return path
+}
+
+func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error) {
+	inputFilePath, err := filepath.Abs(inputFilePath)
+	if err == nil {
+		outputFilePath, err = filepath.Abs(outputFilePath)
+	}
+
+	return inputFilePath, outputFilePath, err
+}
+
+func compress(inPath, outFilePath, subPath string) error {
+	file, err := os.Create(outFilePath)
+	if err != nil {
+		return fmt.Errorf("compress: error creating out file: %w", err)
+	}
+
+	prefix := path.Dir(outFilePath)
+	gzipWriter := gzip.NewWriter(file)
+	tarWriter := tar.NewWriter(gzipWriter)
+
+	var paths []string
+	if err := filepath.WalkDir(inPath, func(path string, di fs.DirEntry, err error) error {
+		paths = append(paths, path)
+		return err
+	}); err != nil {
+		return fmt.Errorf("compress: error walking filesystem tree: %w", err)
+	}
+
+	for _, p := range paths {
+		if err := writeTarGz(p, tarWriter, prefix); err != nil {
+			return fmt.Errorf("compress error writing %s to archive: %w", p, err)
+		}
+	}
+
+	err = tarWriter.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing tar writer: %w", err)
+	}
+
+	err = gzipWriter.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing gzip writer: %w", err)
+	}
+
+	err = file.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing file: %w", err)
+	}
+
+	return nil
+}
+
+func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
+	fileInfo, err := os.Lstat(path)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err)
+	}
+
+	if fileInfo.Mode()&os.ModeSocket == os.ModeSocket {
+		return nil
+	}
+
+	var link string
+	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
+		var err error
+		if link, err = os.Readlink(path); err != nil {
+			return fmt.Errorf("writeTarGz: error resolving symlink %s: %w", path, err)
+		}
+	}
+
+	header, err := tar.FileInfoHeader(fileInfo, link)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error getting file info header: %w", err)
+	}
+	header.Name = strings.TrimPrefix(path, prefix)
+
+	err = tarWriter.WriteHeader(header)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error writing file info header: %w", err)
+	}
+
+	if !fileInfo.Mode().IsRegular() {
+		return nil
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error opening %s: %w", path, err)
+	}
+	defer file.Close()
+
+	_, err = io.Copy(tarWriter, file)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error copying %s to tar writer: %w", path, err)
+	}
+
+	return nil
+}

cmd/backup/config.go

@@ -0,0 +1,45 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import "time"
+
+// Config holds all configuration values that are expected to be set
+// by users.
+type Config struct {
+	BackupSources              string        `split_words:"true" default:"/backup"`
+	BackupFilename             string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
+	BackupFilenameExpand       bool          `split_words:"true"`
+	BackupLatestSymlink        string        `split_words:"true"`
+	BackupArchive              string        `split_words:"true" default:"/archive"`
+	BackupRetentionDays        int32         `split_words:"true" default:"-1"`
+	BackupPruningLeeway        time.Duration `split_words:"true" default:"1m"`
+	BackupPruningPrefix        string        `split_words:"true"`
+	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
+	BackupFromSnapshot         bool          `split_words:"true"`
+	AwsS3BucketName            string        `split_words:"true"`
+	AwsS3Path                  string        `split_words:"true"`
+	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
+	AwsEndpointProto           string        `split_words:"true" default:"https"`
+	AwsEndpointInsecure        bool          `split_words:"true"`
+	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
+	AwsSecretAccessKey         string        `split_words:"true"`
+	AwsIamRoleEndpoint         string        `split_words:"true"`
+	GpgPassphrase              string        `split_words:"true"`
+	NotificationURLs           []string      `envconfig:"NOTIFICATION_URLS"`
+	NotificationLevel          string        `split_words:"true" default:"error"`
+	EmailNotificationRecipient string        `split_words:"true"`
+	EmailNotificationSender    string        `split_words:"true" default:"noreply@nohost"`
+	EmailSMTPHost              string        `envconfig:"EMAIL_SMTP_HOST"`
+	EmailSMTPPort              int           `envconfig:"EMAIL_SMTP_PORT" default:"587"`
+	EmailSMTPUsername          string        `envconfig:"EMAIL_SMTP_USERNAME"`
+	EmailSMTPPassword          string        `envconfig:"EMAIL_SMTP_PASSWORD"`
+	WebdavUrl                  string        `split_words:"true"`
+	WebdavPath                 string        `split_words:"true" default:"/"`
+	WebdavUsername             string        `split_words:"true"`
+	WebdavPassword             string        `split_words:"true"`
+	ExecLabel                  string        `split_words:"true"`
+	ExecForwardOutput          bool          `split_words:"true"`
+	LockTimeout                time.Duration `split_words:"true" default:"60m"`
+}

cmd/backup/exec.go

@@ -0,0 +1,123 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+// Portions of this file are taken and adapted from `moby`, Copyright 2012-2017 Docker, Inc.
+// Licensed under the Apache 2.0 License: https://github.com/moby/moby/blob/8e610b2b55bfd1bfa9436ab110d311f5e8a74dcb/LICENSE
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/cosiner/argv"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/pkg/stdcopy"
+	"golang.org/x/sync/errgroup"
+)
+
+func (s *script) exec(containerRef string, command string) ([]byte, []byte, error) {
+	args, _ := argv.Argv(command, nil, nil)
+	execID, err := s.cli.ContainerExecCreate(context.Background(), containerRef, types.ExecConfig{
+		Cmd:          args[0],
+		AttachStdin:  true,
+		AttachStderr: true,
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error creating container exec: %w", err)
+	}
+
+	resp, err := s.cli.ContainerExecAttach(context.Background(), execID.ID, types.ExecStartCheck{})
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error attaching container exec: %w", err)
+	}
+	defer resp.Close()
+
+	var outBuf, errBuf bytes.Buffer
+	outputDone := make(chan error)
+
+	go func() {
+		_, err := stdcopy.StdCopy(&outBuf, &errBuf, resp.Reader)
+		outputDone <- err
+	}()
+
+	select {
+	case err := <-outputDone:
+		if err != nil {
+			return nil, nil, fmt.Errorf("exec: error demultiplexing output: %w", err)
+		}
+		break
+	}
+
+	stdout, err := ioutil.ReadAll(&outBuf)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error reading stdout: %w", err)
+	}
+	stderr, err := ioutil.ReadAll(&errBuf)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error reading stderr: %w", err)
+	}
+
+	res, err := s.cli.ContainerExecInspect(context.Background(), execID.ID)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error inspecting container exec: %w", err)
+	}
+
+	if res.ExitCode > 0 {
+		return stdout, stderr, fmt.Errorf("exec: running command exited %d", res.ExitCode)
+	}
+
+	return stdout, stderr, nil
+}
+
+func (s *script) runLabeledCommands(label string) error {
+	f := []filters.KeyValuePair{
+		{Key: "label", Value: label},
+	}
+	if s.c.ExecLabel != "" {
+		f = append(f, filters.KeyValuePair{
+			Key:   "label",
+			Value: fmt.Sprintf("docker-volume-backup.exec-label=%s", s.c.ExecLabel),
+		})
+	}
+	containersWithCommand, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Quiet:   true,
+		Filters: filters.NewArgs(f...),
+	})
+	if err != nil {
+		return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
+	}
+
+	if len(containersWithCommand) == 0 {
+		return nil
+	}
+
+	g := new(errgroup.Group)
+
+	for _, container := range containersWithCommand {
+		c := container
+		g.Go(func() error {
+			cmd, _ := c.Labels[label]
+			s.logger.Infof("Running %s command %s for container %s", label, cmd, strings.TrimPrefix(c.Names[0], "/"))
+			stdout, stderr, err := s.exec(c.ID, cmd)
+			if s.c.ExecForwardOutput {
+				os.Stderr.Write(stderr)
+				os.Stdout.Write(stdout)
+			}
+			if err != nil {
+				return fmt.Errorf("runLabeledCommands: error executing command: %w", err)
+			}
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		return fmt.Errorf("runLabeledCommands: error from errgroup: %w", err)
+	}
+	return nil
+}

cmd/backup/hooks.go

@@ -0,0 +1,56 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"fmt"
+	"sort"
+)
+
+// hook contains a queued action that can be trigger them when the script
+// reaches a certain point (e.g. unsuccessful backup)
+type hook struct {
+	level  hookLevel
+	action func(err error) error
+}
+
+type hookLevel int
+
+const (
+	hookLevelPlumbing hookLevel = iota
+	hookLevelError
+	hookLevelInfo
+)
+
+var hookLevels = map[string]hookLevel{
+	"info":  hookLevelInfo,
+	"error": hookLevelError,
+}
+
+// registerHook adds the given action at the given level.
+func (s *script) registerHook(level hookLevel, action func(err error) error) {
+	s.hooks = append(s.hooks, hook{level, action})
+}
+
+// runHooks runs all hooks that have been registered using the
+// given levels in the defined ordering. In case executing a hook returns an
+// error, the following hooks will still be run before the function returns.
+func (s *script) runHooks(err error) error {
+	sort.SliceStable(s.hooks, func(i, j int) bool {
+		return s.hooks[i].level < s.hooks[j].level
+	})
+	var actionErrors []error
+	for _, hook := range s.hooks {
+		if hook.level > s.hookLevel {
+			continue
+		}
+		if actionErr := hook.action(err); actionErr != nil {
+			actionErrors = append(actionErrors, fmt.Errorf("runHooks: error running hook: %w", actionErr))
+		}
+	}
+	if len(actionErrors) != 0 {
+		return join(actionErrors...)
+	}
+	return nil
+}

cmd/backup/lock.go

@@ -0,0 +1,58 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/gofrs/flock"
+)
+
+// lock opens a lockfile at the given location, keeping it locked until the
+// caller invokes the returned release func. In case the lock is currently blocked
+// by another execution, it will repeatedly retry until the lock is available
+// or the given timeout is exceeded.
+func (s *script) lock(lockfile string) (func() error, error) {
+	start := time.Now()
+	defer func() {
+		s.stats.LockedTime = time.Now().Sub(start)
+	}()
+
+	retry := time.NewTicker(5 * time.Second)
+	defer retry.Stop()
+	deadline := time.NewTimer(s.c.LockTimeout)
+	defer deadline.Stop()
+
+	fileLock := flock.New(lockfile)
+
+	for {
+		acquired, err := fileLock.TryLock()
+		if err != nil {
+			return noop, fmt.Errorf("lock: error trying lock: %w", err)
+		}
+		if acquired {
+			if s.encounteredLock {
+				s.logger.Info("Acquired exclusive lock on subsequent attempt, ready to continue.")
+			}
+			return fileLock.Unlock, nil
+		}
+
+		if !s.encounteredLock {
+			s.logger.Infof(
+				"Exclusive lock was not available on first attempt. Will retry until it becomes available or the timeout of %s is exceeded.",
+				s.c.LockTimeout,
+			)
+			s.encounteredLock = true
+		}
+
+		select {
+		case <-retry.C:
+			continue
+		case <-deadline.C:
+			return noop, errors.New("lock: timed out waiting for lockfile to become available")
+		}
+	}
+}

cmd/backup/main.go

@@ -0,0 +1,65 @@
+// Copyright 2021-2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"os"
+)
+
+func main() {
+	s, err := newScript()
+	if err != nil {
+		panic(err)
+	}
+
+	unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
+	defer unlock()
+	s.must(err)
+
+	defer func() {
+		if pArg := recover(); pArg != nil {
+			if err, ok := pArg.(error); ok {
+				if hookErr := s.runHooks(err); hookErr != nil {
+					s.logger.Errorf("An error occurred calling the registered hooks: %s", hookErr)
+				}
+				os.Exit(1)
+			}
+			panic(pArg)
+		}
+
+		if err := s.runHooks(nil); err != nil {
+			s.logger.Errorf(
+				"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
+				err,
+			)
+			os.Exit(1)
+		}
+		s.logger.Info("Finished running backup tasks.")
+	}()
+
+	s.must(func() error {
+		runPostCommands, err := s.runCommands()
+		defer func() {
+			s.must(runPostCommands())
+		}()
+		if err != nil {
+			return err
+		}
+		restartContainers, err := s.stopContainers()
+		// The mechanism for restarting containers is not using hooks as it
+		// should happen as soon as possible (i.e. before uploading backups or
+		// similar).
+		defer func() {
+			s.must(restartContainers())
+		}()
+		if err != nil {
+			return err
+		}
+		return s.takeBackup()
+	}())
+
+	s.must(s.encryptBackup())
+	s.must(s.copyBackup())
+	s.must(s.pruneBackups())
+}

cmd/backup/notifications.go

@@ -0,0 +1,105 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"text/template"
+	"time"
+
+	sTypes "github.com/containrrr/shoutrrr/pkg/types"
+)
+
+//go:embed notifications.tmpl
+var defaultNotifications string
+
+// NotificationData data to be passed to the notification templates
+type NotificationData struct {
+	Error  error
+	Config *Config
+	Stats  *Stats
+}
+
+// notify sends a notification using the given title and body templates.
+// Automatically creates notification data, adding the given error
+func (s *script) notify(titleTemplate string, bodyTemplate string, err error) error {
+	params := NotificationData{
+		Error:  err,
+		Stats:  s.stats,
+		Config: s.c,
+	}
+
+	titleBuf := &bytes.Buffer{}
+	if err := s.template.ExecuteTemplate(titleBuf, titleTemplate, params); err != nil {
+		return fmt.Errorf("notifyFailure: error executing %s template: %w", titleTemplate, err)
+	}
+
+	bodyBuf := &bytes.Buffer{}
+	if err := s.template.ExecuteTemplate(bodyBuf, bodyTemplate, params); err != nil {
+		return fmt.Errorf("notifyFailure: error executing %s template: %w", bodyTemplate, err)
+	}
+
+	if err := s.sendNotification(titleBuf.String(), bodyBuf.String()); err != nil {
+		return fmt.Errorf("notifyFailure: error notifying: %w", err)
+	}
+	return nil
+}
+
+// notifyFailure sends a notification about a failed backup run
+func (s *script) notifyFailure(err error) error {
+	return s.notify("title_failure", "body_failure", err)
+}
+
+// notifyFailure sends a notification about a successful backup run
+func (s *script) notifySuccess() error {
+	return s.notify("title_success", "body_success", nil)
+}
+
+// sendNotification sends a notification to all configured third party services
+func (s *script) sendNotification(title, body string) error {
+	var errs []error
+	for _, result := range s.sender.Send(body, &sTypes.Params{"title": title}) {
+		if result != nil {
+			errs = append(errs, result)
+		}
+	}
+	if len(errs) != 0 {
+		return fmt.Errorf("sendNotification: error sending message: %w", join(errs...))
+	}
+	return nil
+}
+
+var templateHelpers = template.FuncMap{
+	"formatTime": func(t time.Time) string {
+		return t.Format(time.RFC3339)
+	},
+	"formatBytesDec": func(bytes uint64) string {
+		return formatBytes(bytes, true)
+	},
+	"formatBytesBin": func(bytes uint64) string {
+		return formatBytes(bytes, false)
+	},
+}
+
+// formatBytes converts an amount of bytes in a human-readable representation
+// the decimal parameter specifies if using powers of 1000 (decimal) or powers of 1024 (binary)
+func formatBytes(b uint64, decimal bool) string {
+	unit := uint64(1024)
+	format := "%.1f %ciB"
+	if decimal {
+		unit = uint64(1000)
+		format = "%.1f %cB"
+	}
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := unit, 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf(format, float64(b)/float64(div), "kMGTPE"[exp])
+}

cmd/backup/notifications.tmpl

@@ -0,0 +1,26 @@
+{{ define "title_failure" -}}
+Failure running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
+{{- end }}
+
+
+{{ define "body_failure" -}}
+Running docker-volume-backup failed with error: {{ .Error }}
+
+Log output of the failed run was:
+
+{{ .Stats.LogOutput }}
+{{- end }}
+
+
+{{ define "title_success" -}}
+Success running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
+{{- end }}
+
+
+{{ define "body_success" -}}
+Running docker-volume-backup succeeded.
+
+Log output was:
+
+{{ .Stats.LogOutput }}
+{{- end }}

cmd/backup/script.go

@@ -0,0 +1,694 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/containrrr/shoutrrr"
+	"github.com/containrrr/shoutrrr/pkg/router"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/leekchan/timeutil"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/otiai10/copy"
+	"github.com/sirupsen/logrus"
+	"github.com/studio-b12/gowebdav"
+	"golang.org/x/crypto/openpgp"
+)
+
+// script holds all the stateful information required to orchestrate a
+// single backup run.
+type script struct {
+	cli          *client.Client
+	minioClient  *minio.Client
+	webdavClient *gowebdav.Client
+	logger       *logrus.Logger
+	sender       *router.ServiceRouter
+	template     *template.Template
+	hooks        []hook
+	hookLevel    hookLevel
+
+	file  string
+	stats *Stats
+
+	encounteredLock bool
+
+	c *Config
+}
+
+// newScript creates all resources needed for the script to perform actions against
+// remote resources like the Docker engine or remote storage locations. All
+// reading from env vars or other configuration sources is expected to happen
+// in this method.
+func newScript() (*script, error) {
+	stdOut, logBuffer := buffer(os.Stdout)
+	s := &script{
+		c: &Config{},
+		logger: &logrus.Logger{
+			Out:       stdOut,
+			Formatter: new(logrus.TextFormatter),
+			Hooks:     make(logrus.LevelHooks),
+			Level:     logrus.InfoLevel,
+		},
+		stats: &Stats{
+			StartTime: time.Now(),
+			LogOutput: logBuffer,
+			Storages:  StoragesStats{},
+		},
+	}
+
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		s.stats.EndTime = time.Now()
+		s.stats.TookTime = s.stats.EndTime.Sub(s.stats.StartTime)
+		return nil
+	})
+
+	if err := envconfig.Process("", s.c); err != nil {
+		return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
+	}
+
+	s.file = path.Join("/tmp", s.c.BackupFilename)
+	if s.c.BackupFilenameExpand {
+		s.file = os.ExpandEnv(s.file)
+		s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
+		s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix)
+	}
+	s.file = timeutil.Strftime(&s.stats.StartTime, s.file)
+
+	_, err := os.Stat("/var/run/docker.sock")
+	_, dockerHostSet := os.LookupEnv("DOCKER_HOST")
+	if !os.IsNotExist(err) || dockerHostSet {
+		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+		if err != nil {
+			return nil, fmt.Errorf("newScript: failed to create docker client")
+		}
+		s.cli = cli
+	}
+
+	if s.c.AwsS3BucketName != "" {
+		var creds *credentials.Credentials
+		if s.c.AwsAccessKeyID != "" && s.c.AwsSecretAccessKey != "" {
+			creds = credentials.NewStaticV4(
+				s.c.AwsAccessKeyID,
+				s.c.AwsSecretAccessKey,
+				"",
+			)
+		} else if s.c.AwsIamRoleEndpoint != "" {
+			creds = credentials.NewIAM(s.c.AwsIamRoleEndpoint)
+		} else {
+			return nil, errors.New("newScript: AWS_S3_BUCKET_NAME is defined, but no credentials were provided")
+		}
+
+		options := minio.Options{
+			Creds:  creds,
+			Secure: s.c.AwsEndpointProto == "https",
+		}
+
+		if s.c.AwsEndpointInsecure {
+			if !options.Secure {
+				return nil, errors.New("newScript: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
+			}
+
+			transport, err := minio.DefaultTransport(true)
+			if err != nil {
+				return nil, fmt.Errorf("newScript: failed to create default minio transport")
+			}
+			transport.TLSClientConfig.InsecureSkipVerify = true
+			options.Transport = transport
+		}
+
+		mc, err := minio.New(s.c.AwsEndpoint, &options)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: error setting up minio client: %w", err)
+		}
+		s.minioClient = mc
+	}
+
+	if s.c.WebdavUrl != "" {
+		if s.c.WebdavUsername == "" || s.c.WebdavPassword == "" {
+			return nil, errors.New("newScript: WEBDAV_URL is defined, but no credentials were provided")
+		} else {
+			webdavClient := gowebdav.NewClient(s.c.WebdavUrl, s.c.WebdavUsername, s.c.WebdavPassword)
+			s.webdavClient = webdavClient
+		}
+	}
+
+	if s.c.EmailNotificationRecipient != "" {
+		emailURL := fmt.Sprintf(
+			"smtp://%s:%s@%s:%d/?from=%s&to=%s",
+			s.c.EmailSMTPUsername,
+			s.c.EmailSMTPPassword,
+			s.c.EmailSMTPHost,
+			s.c.EmailSMTPPort,
+			s.c.EmailNotificationSender,
+			s.c.EmailNotificationRecipient,
+		)
+		s.c.NotificationURLs = append(s.c.NotificationURLs, emailURL)
+		s.logger.Warn(
+			"Using EMAIL_* keys for providing notification configuration has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use NOTIFICATION_URLS instead. Refer to the README for an upgrade guide.",
+		)
+	}
+
+	hookLevel, ok := hookLevels[s.c.NotificationLevel]
+	if !ok {
+		return nil, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel)
+	}
+	s.hookLevel = hookLevel
+
+	if len(s.c.NotificationURLs) > 0 {
+		sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...)
+		if senderErr != nil {
+			return nil, fmt.Errorf("newScript: error creating sender: %w", senderErr)
+		}
+		s.sender = sender
+
+		tmpl := template.New("")
+		tmpl.Funcs(templateHelpers)
+		tmpl, err = tmpl.Parse(defaultNotifications)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
+		}
+
+		if fi, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil && fi.IsDir() {
+			tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*")
+			if err != nil {
+				return nil, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
+			}
+		}
+		s.template = tmpl
+
+		// To prevent duplicate notifications, ensure the regsistered callbacks
+		// run mutually exclusive.
+		s.registerHook(hookLevelError, func(err error) error {
+			if err == nil {
+				return nil
+			}
+			return s.notifyFailure(err)
+		})
+		s.registerHook(hookLevelInfo, func(err error) error {
+			if err != nil {
+				return nil
+			}
+			return s.notifySuccess()
+		})
+	}
+
+	return s, nil
+}
+
+func (s *script) runCommands() (func() error, error) {
+	if s.cli == nil {
+		return noop, nil
+	}
+
+	if err := s.runLabeledCommands("docker-volume-backup.exec-pre"); err != nil {
+		return noop, fmt.Errorf("runCommands: error running pre commands: %w", err)
+	}
+	return func() error {
+		if err := s.runLabeledCommands("docker-volume-backup.exec-post"); err != nil {
+			return fmt.Errorf("runCommands: error running post commands: %w", err)
+		}
+		return nil
+	}, nil
+}
+
+// stopContainers stops all Docker containers that are marked as to being
+// stopped during the backup and returns a function that can be called to
+// restart everything that has been stopped.
+func (s *script) stopContainers() (func() error, error) {
+	if s.cli == nil {
+		return noop, nil
+	}
+
+	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Quiet: true,
+	})
+	if err != nil {
+		return noop, fmt.Errorf("stopContainersAndRun: error querying for containers: %w", err)
+	}
+
+	containerLabel := fmt.Sprintf(
+		"docker-volume-backup.stop-during-backup=%s",
+		s.c.BackupStopContainerLabel,
+	)
+	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Quiet: true,
+		Filters: filters.NewArgs(filters.KeyValuePair{
+			Key:   "label",
+			Value: containerLabel,
+		}),
+	})
+
+	if err != nil {
+		return noop, fmt.Errorf("stopContainersAndRun: error querying for containers to stop: %w", err)
+	}
+
+	if len(containersToStop) == 0 {
+		return noop, nil
+	}
+
+	s.logger.Infof(
+		"Stopping %d container(s) labeled `%s` out of %d running container(s).",
+		len(containersToStop),
+		containerLabel,
+		len(allContainers),
+	)
+
+	var stoppedContainers []types.Container
+	var stopErrors []error
+	for _, container := range containersToStop {
+		if err := s.cli.ContainerStop(context.Background(), container.ID, nil); err != nil {
+			stopErrors = append(stopErrors, err)
+		} else {
+			stoppedContainers = append(stoppedContainers, container)
+		}
+	}
+
+	var stopError error
+	if len(stopErrors) != 0 {
+		stopError = fmt.Errorf(
+			"stopContainersAndRun: %d error(s) stopping containers: %w",
+			len(stopErrors),
+			join(stopErrors...),
+		)
+	}
+
+	s.stats.Containers = ContainersStats{
+		All:     uint(len(allContainers)),
+		ToStop:  uint(len(containersToStop)),
+		Stopped: uint(len(stoppedContainers)),
+	}
+
+	return func() error {
+		servicesRequiringUpdate := map[string]struct{}{}
+
+		var restartErrors []error
+		for _, container := range stoppedContainers {
+			if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok {
+				servicesRequiringUpdate[swarmServiceName] = struct{}{}
+				continue
+			}
+			if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil {
+				restartErrors = append(restartErrors, err)
+			}
+		}
+
+		if len(servicesRequiringUpdate) != 0 {
+			services, _ := s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
+			for serviceName := range servicesRequiringUpdate {
+				var serviceMatch swarm.Service
+				for _, service := range services {
+					if service.Spec.Name == serviceName {
+						serviceMatch = service
+						break
+					}
+				}
+				if serviceMatch.ID == "" {
+					return fmt.Errorf("stopContainersAndRun: couldn't find service with name %s", serviceName)
+				}
+				serviceMatch.Spec.TaskTemplate.ForceUpdate = 1
+				if _, err := s.cli.ServiceUpdate(
+					context.Background(), serviceMatch.ID,
+					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},
+				); err != nil {
+					restartErrors = append(restartErrors, err)
+				}
+			}
+		}
+
+		if len(restartErrors) != 0 {
+			return fmt.Errorf(
+				"stopContainersAndRun: %d error(s) restarting containers and services: %w",
+				len(restartErrors),
+				join(restartErrors...),
+			)
+		}
+		s.logger.Infof(
+			"Restarted %d container(s) and the matching service(s).",
+			len(stoppedContainers),
+		)
+		return nil
+	}, stopError
+}
+
+// takeBackup creates a tar archive of the configured backup location and
+// saves it to disk.
+func (s *script) takeBackup() error {
+	backupSources := s.c.BackupSources
+
+	if s.c.BackupFromSnapshot {
+		s.logger.Warn(
+			"Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use `exec-pre` and `exec-post` commands to prepare your backup sources. Refer to the README for an upgrade guide.",
+		)
+		backupSources = filepath.Join("/tmp", s.c.BackupSources)
+		// copy before compressing guard against a situation where backup folder's content are still growing.
+		s.registerHook(hookLevelPlumbing, func(error) error {
+			if err := remove(backupSources); err != nil {
+				return fmt.Errorf("takeBackup: error removing snapshot: %w", err)
+			}
+			s.logger.Infof("Removed snapshot `%s`.", backupSources)
+			return nil
+		})
+		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
+			PreserveTimes: true,
+			PreserveOwner: true,
+		}); err != nil {
+			return fmt.Errorf("takeBackup: error creating snapshot: %w", err)
+		}
+		s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources)
+	}
+
+	tarFile := s.file
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		if err := remove(tarFile); err != nil {
+			return fmt.Errorf("takeBackup: error removing tar file: %w", err)
+		}
+		s.logger.Infof("Removed tar file `%s`.", tarFile)
+		return nil
+	})
+	if err := createArchive(backupSources, tarFile); err != nil {
+		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
+	}
+
+	s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, tarFile)
+	return nil
+}
+
+// encryptBackup encrypts the backup file using PGP and the configured passphrase.
+// In case no passphrase is given it returns early, leaving the backup file
+// untouched.
+func (s *script) encryptBackup() error {
+	if s.c.GpgPassphrase == "" {
+		return nil
+	}
+
+	gpgFile := fmt.Sprintf("%s.gpg", s.file)
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		if err := remove(gpgFile); err != nil {
+			return fmt.Errorf("encryptBackup: error removing gpg file: %w", err)
+		}
+		s.logger.Infof("Removed GPG file `%s`.", gpgFile)
+		return nil
+	})
+
+	outFile, err := os.Create(gpgFile)
+	defer outFile.Close()
+	if err != nil {
+		return fmt.Errorf("encryptBackup: error opening out file: %w", err)
+	}
+
+	_, name := path.Split(s.file)
+	dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
+		IsBinary: true,
+		FileName: name,
+	}, nil)
+	defer dst.Close()
+	if err != nil {
+		return fmt.Errorf("encryptBackup: error encrypting backup file: %w", err)
+	}
+
+	src, err := os.Open(s.file)
+	if err != nil {
+		return fmt.Errorf("encryptBackup: error opening backup file `%s`: %w", s.file, err)
+	}
+
+	if _, err := io.Copy(dst, src); err != nil {
+		return fmt.Errorf("encryptBackup: error writing ciphertext to file: %w", err)
+	}
+
+	s.file = gpgFile
+	s.logger.Infof("Encrypted backup using given passphrase, saving as `%s`.", s.file)
+	return nil
+}
+
+// copyBackup makes sure the backup file is copied to both local and remote locations
+// as per the given configuration.
+func (s *script) copyBackup() error {
+	_, name := path.Split(s.file)
+	if stat, err := os.Stat(s.file); err != nil {
+		return fmt.Errorf("copyBackup: unable to stat backup file: %w", err)
+	} else {
+		size := stat.Size()
+		s.stats.BackupFile = BackupFileStats{
+			Size:     uint64(size),
+			Name:     name,
+			FullPath: s.file,
+		}
+	}
+
+	if s.minioClient != nil {
+		if _, err := s.minioClient.FPutObject(context.Background(), s.c.AwsS3BucketName, filepath.Join(s.c.AwsS3Path, name), s.file, minio.PutObjectOptions{
+			ContentType: "application/tar+gzip",
+		}); err != nil {
+			return fmt.Errorf("copyBackup: error uploading backup to remote storage: %w", err)
+		}
+		s.logger.Infof("Uploaded a copy of backup `%s` to bucket `%s`.", s.file, s.c.AwsS3BucketName)
+	}
+
+	if s.webdavClient != nil {
+		bytes, err := os.ReadFile(s.file)
+		if err != nil {
+			return fmt.Errorf("copyBackup: error reading the file to be uploaded: %w", err)
+		}
+		if err := s.webdavClient.MkdirAll(s.c.WebdavPath, 0644); err != nil {
+			return fmt.Errorf("copyBackup: error creating directory '%s' on WebDAV server: %w", s.c.WebdavPath, err)
+		}
+		if err := s.webdavClient.Write(filepath.Join(s.c.WebdavPath, name), bytes, 0644); err != nil {
+			return fmt.Errorf("copyBackup: error uploading the file to WebDAV server: %w", err)
+		}
+		s.logger.Infof("Uploaded a copy of backup `%s` to WebDAV-URL '%s' at path '%s'.", s.file, s.c.WebdavUrl, s.c.WebdavPath)
+	}
+
+	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
+		if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
+			return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
+		}
+		s.logger.Infof("Stored copy of backup `%s` in local archive `%s`.", s.file, s.c.BackupArchive)
+		if s.c.BackupLatestSymlink != "" {
+			symlink := path.Join(s.c.BackupArchive, s.c.BackupLatestSymlink)
+			if _, err := os.Lstat(symlink); err == nil {
+				os.Remove(symlink)
+			}
+			if err := os.Symlink(name, symlink); err != nil {
+				return fmt.Errorf("copyBackup: error creating latest symlink: %w", err)
+			}
+			s.logger.Infof("Created/Updated symlink `%s` for latest backup.", s.c.BackupLatestSymlink)
+		}
+	}
+	return nil
+}
+
+// pruneBackups rotates away backups from local and remote storages using
+// the given configuration. In case the given configuration would delete all
+// backups, it does nothing instead and logs a warning.
+func (s *script) pruneBackups() error {
+	if s.c.BackupRetentionDays < 0 {
+		return nil
+	}
+
+	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
+
+	// doPrune holds general control flow that applies to any kind of storage.
+	// Callers can pass in a thunk that performs the actual deletion of files.
+	var doPrune = func(lenMatches, lenCandidates int, description string, doRemoveFiles func() error) error {
+		if lenMatches != 0 && lenMatches != lenCandidates {
+			if err := doRemoveFiles(); err != nil {
+				return err
+			}
+			s.logger.Infof(
+				"Pruned %d out of %d %s as their age exceeded the configured retention period of %d days.",
+				lenMatches,
+				lenCandidates,
+				description,
+				s.c.BackupRetentionDays,
+			)
+		} else if lenMatches != 0 && lenMatches == lenCandidates {
+			s.logger.Warnf("The current configuration would delete all %d existing %s.", lenMatches, description)
+			s.logger.Warn("Refusing to do so, please check your configuration.")
+		} else {
+			s.logger.Infof("None of %d existing %s were pruned.", lenCandidates, description)
+		}
+		return nil
+	}
+
+	if s.minioClient != nil {
+		candidates := s.minioClient.ListObjects(context.Background(), s.c.AwsS3BucketName, minio.ListObjectsOptions{
+			WithMetadata: true,
+			Prefix:       filepath.Join(s.c.AwsS3Path, s.c.BackupPruningPrefix),
+			Recursive:    true,
+		})
+
+		var matches []minio.ObjectInfo
+		var lenCandidates int
+		for candidate := range candidates {
+			lenCandidates++
+			if candidate.Err != nil {
+				return fmt.Errorf(
+					"pruneBackups: error looking up candidates from remote storage: %w",
+					candidate.Err,
+				)
+			}
+			if candidate.LastModified.Before(deadline) {
+				matches = append(matches, candidate)
+			}
+		}
+
+		s.stats.Storages.S3 = StorageStats{
+			Total:  uint(lenCandidates),
+			Pruned: uint(len(matches)),
+		}
+
+		doPrune(len(matches), lenCandidates, "remote backup(s)", func() error {
+			objectsCh := make(chan minio.ObjectInfo)
+			go func() {
+				for _, match := range matches {
+					objectsCh <- match
+				}
+				close(objectsCh)
+			}()
+			errChan := s.minioClient.RemoveObjects(context.Background(), s.c.AwsS3BucketName, objectsCh, minio.RemoveObjectsOptions{})
+			var removeErrors []error
+			for result := range errChan {
+				if result.Err != nil {
+					removeErrors = append(removeErrors, result.Err)
+				}
+			}
+			if len(removeErrors) != 0 {
+				return join(removeErrors...)
+			}
+			return nil
+		})
+	}
+
+	if s.webdavClient != nil {
+		candidates, err := s.webdavClient.ReadDir(s.c.WebdavPath)
+		if err != nil {
+			return fmt.Errorf("pruneBackups: error looking up candidates from remote storage: %w", err)
+		}
+		var matches []fs.FileInfo
+		var lenCandidates int
+		for _, candidate := range candidates {
+			if !strings.HasPrefix(candidate.Name(), s.c.BackupPruningPrefix) {
+				continue
+			}
+			lenCandidates++
+			if candidate.ModTime().Before(deadline) {
+				matches = append(matches, candidate)
+			}
+		}
+
+		s.stats.Storages.WebDAV = StorageStats{
+			Total:  uint(lenCandidates),
+			Pruned: uint(len(matches)),
+		}
+
+		doPrune(len(matches), lenCandidates, "WebDAV backup(s)", func() error {
+			for _, match := range matches {
+				if err := s.webdavClient.Remove(filepath.Join(s.c.WebdavPath, match.Name())); err != nil {
+					return fmt.Errorf("pruneBackups: error removing file from WebDAV storage: %w", err)
+				}
+			}
+			return nil
+		})
+	}
+
+	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
+		globPattern := path.Join(
+			s.c.BackupArchive,
+			fmt.Sprintf("%s*", s.c.BackupPruningPrefix),
+		)
+		globMatches, err := filepath.Glob(globPattern)
+		if err != nil {
+			return fmt.Errorf(
+				"pruneBackups: error looking up matching files using pattern %s: %w",
+				globPattern,
+				err,
+			)
+		}
+
+		var candidates []string
+		for _, candidate := range globMatches {
+			fi, err := os.Lstat(candidate)
+			if err != nil {
+				return fmt.Errorf(
+					"pruneBackups: error calling Lstat on file %s: %w",
+					candidate,
+					err,
+				)
+			}
+
+			if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+				candidates = append(candidates, candidate)
+			}
+		}
+
+		var matches []string
+		for _, candidate := range candidates {
+			fi, err := os.Stat(candidate)
+			if err != nil {
+				return fmt.Errorf(
+					"pruneBackups: error calling stat on file %s: %w",
+					candidate,
+					err,
+				)
+			}
+			if fi.ModTime().Before(deadline) {
+				matches = append(matches, candidate)
+			}
+		}
+
+		s.stats.Storages.Local = StorageStats{
+			Total:  uint(len(candidates)),
+			Pruned: uint(len(matches)),
+		}
+
+		doPrune(len(matches), len(candidates), "local backup(s)", func() error {
+			var removeErrors []error
+			for _, match := range matches {
+				if err := os.Remove(match); err != nil {
+					removeErrors = append(removeErrors, err)
+				}
+			}
+			if len(removeErrors) != 0 {
+				return fmt.Errorf(
+					"pruneBackups: %d error(s) deleting local files, starting with: %w",
+					len(removeErrors),
+					join(removeErrors...),
+				)
+			}
+			return nil
+		})
+	}
+	return nil
+}
+
+// must exits the script run prematurely in case the given error
+// is non-nil.
+func (s *script) must(err error) {
+	if err != nil {
+		s.logger.Errorf("Fatal error running backup: %s", err)
+		panic(err)
+	}
+}

cmd/backup/stats.go

@@ -0,0 +1,50 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	"time"
+)
+
+// ContainersStats stats about the docker containers
+type ContainersStats struct {
+	All        uint
+	ToStop     uint
+	Stopped    uint
+	StopErrors uint
+}
+
+// BackupFileStats stats about the created backup file
+type BackupFileStats struct {
+	Name     string
+	FullPath string
+	Size     uint64
+}
+
+// StorageStats stats about the status of an archival directory
+type StorageStats struct {
+	Total       uint
+	Pruned      uint
+	PruneErrors uint
+}
+
+// StoragesStats stats about each possible archival location (Local, WebDAV, S3)
+type StoragesStats struct {
+	Local  StorageStats
+	WebDAV StorageStats
+	S3     StorageStats
+}
+
+// Stats global stats regarding script execution
+type Stats struct {
+	StartTime  time.Time
+	EndTime    time.Time
+	TookTime   time.Duration
+	LockedTime time.Duration
+	LogOutput  *bytes.Buffer
+	Containers ContainersStats
+	BackupFile BackupFileStats
+	Storages   StoragesStats
+}

cmd/backup/util.go

@@ -0,0 +1,90 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+var noop = func() error { return nil }
+
+// copy creates a copy of the file located at `dst` at `src`.
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}
+
+// join takes a list of errors and joins them into a single error
+func join(errs ...error) error {
+	if len(errs) == 1 {
+		return errs[0]
+	}
+	var msgs []string
+	for _, err := range errs {
+		if err == nil {
+			continue
+		}
+		msgs = append(msgs, err.Error())
+	}
+	return errors.New("[" + strings.Join(msgs, ", ") + "]")
+}
+
+// remove removes the given file or directory from disk.
+func remove(location string) error {
+	fi, err := os.Lstat(location)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("remove: error checking for existence of `%s`: %w", location, err)
+	}
+	if fi.IsDir() {
+		err = os.RemoveAll(location)
+	} else {
+		err = os.Remove(location)
+	}
+	if err != nil {
+		return fmt.Errorf("remove: error removing `%s`: %w", location, err)
+	}
+	return nil
+}
+
+// buffer takes an io.Writer and returns a wrapped version of the
+// writer that writes to both the original target as well as the returned buffer
+func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {
+	buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w}
+	return buffering, &buffering.buf
+}
+
+type bufferingWriter struct {
+	buf    bytes.Buffer
+	writer io.Writer
+}
+
+func (b *bufferingWriter) Write(p []byte) (n int, err error) {
+	if n, err := b.buf.Write(p); err != nil {
+		return n, fmt.Errorf("bufferingWriter: error writing to buffer: %w", err)
+	}
+	return b.writer.Write(p)
+}

docs/NOTIFICATION-TEMPLATES.md

@@ -0,0 +1,39 @@
+# Notification templates reference
+
+In order to customize title and body of notifications you'll have to write a [go template](https://pkg.go.dev/text/template) and mount it inside the `/etc/dockervolumebackup/notifications.d/` directory.
+
+Configuration, data about the backup run and helper functions will be passed to this template, this page documents them fully.
+
+## Data
+Here is a list of all data passed to the template:
+
+* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`)
+* `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates
+* `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available.
+  * `StartTime`: time when the script started execution
+  * `EndTime`: time when the backup has completed successfully (after pruning)
+  * `TookTime`: amount of time it took for the backup to run. (equal to `EndTime - StartTime`)
+  * `LockedTime`: amount of time it took for the backup to acquire the exclusive lock
+  * `LogOutput`: full log of the application
+  * `Containers`: object containing stats about the docker containers
+    * `All`: total number of containers
+    * `ToStop`: number of containers matched by the stop rule
+    * `Stopped`: number of containers successfully stopped
+    * `StopErrors`: number of containers that were unable to be stopped (equal to `ToStop - Stopped`)
+  * `BackupFile`: object containing information about the backup file
+    * `Name`: name of the backup file (e.g. `backup-2022-02-11T01-00-00.tar.gz`)
+    * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
+    * `Size`: size in bytes of the backup file
+  * `Storages`: object that holds stats about each storage
+    * `Local`, `S3` or `WebDAV`:
+      * `Total`: total number of backup files
+      * `Pruned`: number of backup files that were deleted due to pruning rule
+      * `PruneErrors`: number of backup files that were unable to be pruned
+
+## Functions
+
+Some formatting functions are also available:
+
+* `formatTime`: formats a time object using [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format (e.g. `2022-02-11T01:00:00Z`)
+* `formatBytesBin`: formats an amount of bytes using powers of 1024 (e.g. `7055258` bytes will be `6.7 MiB`) 
+* `formatBytesDec`: formats an amount of bytes using powers of 1000 (e.g. `7055258` bytes will be `7.1 MB`)

entrypoint.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# Copyright 2021 - Offen Authors <hioffen@posteo.de>
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+if [ ! -d "/etc/dockervolumebackup/conf.d" ]; then
+  BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+
+  echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
+  echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
+else
+  echo "/etc/dockervolumebackup/conf.d was found, using configuration files from this directory."
+
+  for file in /etc/dockervolumebackup/conf.d/*; do
+    source $file
+    BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+    echo "Appending cron.d entry with expression $BACKUP_CRON_EXPRESSION and configuration file $file"
+    (crontab -l; echo "$BACKUP_CRON_EXPRESSION /bin/sh -c 'set -a; source $file; set +a && backup' 2>&1") | crontab -
+  done
+fi
+
+echo "Starting cron in foreground."
+crond -f -l 8

go.mod

@@ -0,0 +1,60 @@
+module github.com/offen/docker-volume-backup
+
+go 1.18
+
+require (
+	github.com/containrrr/shoutrrr v0.5.2
+	github.com/cosiner/argv v0.1.0
+	github.com/docker/docker v20.10.11+incompatible
+	github.com/gofrs/flock v0.8.1
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
+	github.com/minio/minio-go/v7 v7.0.16
+	github.com/otiai10/copy v1.7.0
+	github.com/sirupsen/logrus v1.8.1
+	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+)
+
+require (
+	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/containerd/containerd v1.5.5 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/nxadm/tail v1.4.6 // indirect
+	github.com/onsi/ginkgo v1.14.2 // indirect
+	github.com/onsi/gomega v1.10.3 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rs/xid v1.3.0 // indirect
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
+	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
+	google.golang.org/grpc v1.33.2 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/ini.v1 v1.65.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)

src/backup.sh

@@ -1,150 +0,0 @@
-#!/bin/sh
-
-# Copyright 2021 - Offen Authors <hioffen@posteo.de>
-# SPDX-License-Identifier: MPL-2.0
-
-# Portions of this file are taken from github.com/futurice/docker-volume-backup
-# See NOTICE for information about authors and licensing.
-
-source env.sh
-
-function info {
-  echo -e "\n[INFO] $1\n"
-}
-
-info "Preparing backup"
-DOCKER_SOCK="/var/run/docker.sock"
-
-if [ -S "$DOCKER_SOCK" ]; then
-  TEMPFILE="$(mktemp)"
-  docker ps -q \
-    --filter "label=docker-volume-backup.stop-during-backup=$BACKUP_STOP_CONTAINER_LABEL" \
-    > "$TEMPFILE"
-  CONTAINERS_TO_STOP="$(cat $TEMPFILE | tr '\n' ' ')"
-  CONTAINERS_TO_STOP_TOTAL="$(cat $TEMPFILE | wc -l)"
-  CONTAINERS_TOTAL="$(docker ps -q | wc -l)"
-  rm "$TEMPFILE"
-  echo "$CONTAINERS_TOTAL containers running on host in total."
-  echo "$CONTAINERS_TO_STOP_TOTAL containers marked to be stopped during backup."
-else
-  CONTAINERS_TO_STOP_TOTAL="0"
-  CONTAINERS_TOTAL="0"
-  echo "Cannot access \"$DOCKER_SOCK\", won't look for containers to stop."
-fi
-
-if [ "$CONTAINERS_TO_STOP_TOTAL" != "0" ]; then
-  info "Stopping containers"
-  docker stop $CONTAINERS_TO_STOP
-fi
-
-info "Creating backup"
-BACKUP_FILENAME="$(date +"$BACKUP_FILENAME")"
-tar -czvf "$BACKUP_FILENAME" $BACKUP_SOURCES # allow the var to expand, in case we have multiple sources
-
-if [ ! -z "$GPG_PASSPHRASE" ]; then
-  info "Encrypting backup"
-  gpg --symmetric --cipher-algo aes256 --batch --passphrase "$GPG_PASSPHRASE" \
-    -o "${BACKUP_FILENAME}.gpg" $BACKUP_FILENAME
-  rm $BACKUP_FILENAME
-  BACKUP_FILENAME="${BACKUP_FILENAME}.gpg"
-fi
-
-if [ "$CONTAINERS_TO_STOP_TOTAL" != "0" ]; then
-  info "Starting containers/services back up"
-  # The container might be part of a stack when running in swarm mode, so
-  # its parent service needs to be restarted instead once backup is finished.
-  SERVICES_REQUIRING_UPDATE=""
-  for CONTAINER_ID in $CONTAINERS_TO_STOP; do
-    SWARM_SERVICE_NAME=$(
-      docker inspect \
-        --format "{{ index .Config.Labels \"com.docker.swarm.service.name\" }}" \
-        $CONTAINER_ID
-    )
-    if [ -z "$SWARM_SERVICE_NAME" ]; then
-      echo "Restarting $(docker start $CONTAINER_ID)"
-    else
-      echo "Removing $(docker rm $CONTAINER_ID)"
-      # Multiple containers might belong to the same service, so they will
-      # be restarted only after all names are known.
-      SERVICES_REQUIRING_UPDATE="${SERVICES_REQUIRING_UPDATE} ${SWARM_SERVICE_NAME}"
-    fi
-  done
-
-  if [ -n "$SERVICES_REQUIRING_UPDATE" ]; then
-    for SERVICE_NAME in $(echo -n "$SERVICES_REQUIRING_UPDATE" | tr ' ' '\n' | sort -u); do
-      docker service update --force $SERVICE_NAME
-    done
-  fi
-fi
-
-copy_backup () {
-  mc cp $MC_GLOBAL_OPTIONS "$BACKUP_FILENAME" "$1"
-}
-
-if [ ! -z "$AWS_S3_BUCKET_NAME" ]; then
-  info "Uploading backup to remote storage"
-  echo "Will upload to bucket \"$AWS_S3_BUCKET_NAME\"."
-  copy_backup "backup-target/$AWS_S3_BUCKET_NAME"
-  echo "Upload finished."
-fi
-
-if [ -d "$BACKUP_ARCHIVE" ]; then
-  info "Copying backup to local archive"
-  echo "Will copy to \"$BACKUP_ARCHIVE\"."
-  copy_backup "$BACKUP_ARCHIVE"
-  echo "Finished copying."
-fi
-
-if [ -f "$BACKUP_FILENAME" ]; then
-  info "Cleaning up"
-  rm -vf "$BACKUP_FILENAME"
-fi
-
-info "Backup finished"
-echo "Will wait for next scheduled backup."
-
-prune () {
-  target=$1
-  if [ ! -z "$BACKUP_PRUNING_PREFIX" ]; then
-    target="$target/${BACKUP_PRUNING_PREFIX}"
-  fi
-
-  rule_applies_to=$(
-    mc rm $MC_GLOBAL_OPTIONS --fake --recursive --force \
-      --older-than "${BACKUP_RETENTION_DAYS}d" \
-      "$target" \
-      | wc -l
-  )
-  if [ "$rule_applies_to" == "0" ]; then
-    echo "No backups found older than the configured retention period of $BACKUP_RETENTION_DAYS days."
-    echo "Doing nothing."
-    exit 0
-  fi
-
-  total=$(mc ls $MC_GLOBAL_OPTIONS "$target" | wc -l)
-
-  if [ "$rule_applies_to" == "$total" ]; then
-    echo "Using a retention of ${BACKUP_RETENTION_DAYS} days would prune all currently existing backups, will not continue."
-    echo "If this is what you want, please remove files manually instead of using this script."
-    exit 1
-  fi
-
-  mc rm $MC_GLOBAL_OPTIONS \
-    --recursive --force \
-    --older-than "${BACKUP_RETENTION_DAYS}d" "$target"
-  echo "Successfully pruned ${rule_applies_to} backups older than ${BACKUP_RETENTION_DAYS} days."
-}
-
-if [ ! -z "$BACKUP_RETENTION_DAYS" ]; then
-  info "Pruning old backups"
-  echo "Sleeping ${BACKUP_PRUNING_LEEWAY} before checking eligibility."
-  sleep "$BACKUP_PRUNING_LEEWAY"
-  if [ ! -z "$AWS_S3_BUCKET_NAME" ]; then
-    info "Pruning old backups from remote storage"
-    prune "backup-target/$bucket"
-  fi
-  if [ -d "$BACKUP_ARCHIVE" ]; then
-    info "Pruning old backups from local archive"
-    prune "$BACKUP_ARCHIVE"
-  fi
-fi

src/entrypoint.sh

@@ -1,46 +0,0 @@
-#!/bin/sh
-
-# Copyright 2021 - Offen Authors <hioffen@posteo.de>
-# SPDX-License-Identifier: MPL-2.0
-
-# Portions of this file are taken from github.com/futurice/docker-volume-backup
-# See NOTICE for information about authors and licensing.
-
-set -e
-
-# Write cronjob env to file, fill in sensible defaults, and read them back in
-cat <<EOF > env.sh
-BACKUP_SOURCES="${BACKUP_SOURCES:-/backup}"
-BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
-BACKUP_FILENAME="${BACKUP_FILENAME:-backup-%Y-%m-%dT%H-%M-%S.tar.gz}"
-BACKUP_ARCHIVE="${BACKUP_ARCHIVE:-/archive}"
-
-BACKUP_RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-}"
-BACKUP_PRUNING_LEEWAY="${BACKUP_PRUNING_LEEWAY:-10m}"
-BACKUP_PRUNING_PREFIX="${BACKUP_PRUNING_PREFIX:-}"
-BACKUP_STOP_CONTAINER_LABEL="${BACKUP_STOP_CONTAINER_LABEL:-true}"
-
-AWS_S3_BUCKET_NAME="${AWS_S3_BUCKET_NAME:-}"
-AWS_ENDPOINT="${AWS_ENDPOINT:-s3.amazonaws.com}"
-AWS_ENDPOINT_PROTO="${AWS_ENDPOINT_PROTO:-https}"
-
-GPG_PASSPHRASE="${GPG_PASSPHRASE:-}"
-
-MC_GLOBAL_OPTIONS="${MC_GLOBAL_OPTIONS:-}"
-EOF
-chmod a+x env.sh
-source env.sh
-
-if [ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ]; then
-  mc $MC_GLOBAL_OPTIONS alias set backup-target \
-    "$AWS_ENDPOINT_PROTO://$AWS_ENDPOINT" \
-    "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
-fi
-
-# Add our cron entry, and direct stdout & stderr to Docker commands stdout
-echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
-echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
-
-# Let cron take the wheel
-echo "Starting cron in foreground."
-crond -f -l 8

test/cli/run.sh

@@ -7,6 +7,9 @@ cd $(dirname $0)
 docker network create test_network
 docker volume create backup_data
 docker volume create app_data
+# This volume is created to test whether empty directories are handled
+# correctly. It is not supposed to hold any data.
+docker volume create empty_data
 
 docker run -d \
   --name minio \
@@ -23,16 +26,15 @@ docker exec minio mkdir -p /data/backup
 docker run -d \
   --name offen \
   --network test_network \
-  --label "docker-volume-backup.stop-during-backup=true" \
   -v app_data:/var/opt/offen/ \
   offen/offen:latest
 
 sleep 10
 
-docker run -d \
-  --name backup \
+docker run --rm \
   --network test_network \
   -v app_data:/backup/app_data \
+  -v empty_data:/backup/empty_data \
   -v /var/run/docker.sock:/var/run/docker.sock \
   --env AWS_ACCESS_KEY_ID=test \
   --env AWS_SECRET_ACCESS_KEY=GMusLtUmILge2by+z890kQ \
@@ -40,18 +42,19 @@ docker run -d \
   --env AWS_ENDPOINT_PROTO=http \
   --env AWS_S3_BUCKET_NAME=backup \
   --env BACKUP_FILENAME=test.tar.gz \
-  --env BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?" \
-  offen/docker-volume-backup:$TEST_VERSION
-
-docker exec backup backup
+  --env "BACKUP_FROM_SNAPSHOT=true" \
+  --entrypoint backup \
+  offen/docker-volume-backup:${TEST_VERSION:-canary}
 
 docker run --rm -it \
   -v backup_data:/data alpine \
-  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
+  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db && test -d /backup/empty_data'
 
-echo "[TEST:PASS] Found relevant files in untared backup."
+echo "[TEST:PASS] Found relevant files in untared remote backup."
 
-if [ "$(docker ps -q | wc -l)" != "3" ]; then
+# This test does not stop containers during backup. This is happening on
+# purpose in order to cover this setup as well.
+if [ "$(docker ps -q | wc -l)" != "2" ]; then
   echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
   docker ps
   exit 1
@@ -59,6 +62,6 @@ fi
 
 echo "[TEST:PASS] All containers running post backup."
 
-docker rm $(docker stop minio offen backup)
+docker rm $(docker stop minio offen)
 docker volume rm backup_data app_data
 docker network rm test_network

test/commands/.gitignore

@@ -0,0 +1 @@
+local

test/commands/docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3.8'
+
+services:
+  database:
+    image: mariadb:10.7
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: backup
+    labels:
+      - docker-volume-backup.exec-pre=/bin/sh -c 'mysqldump -ptest --all-databases > /tmp/volume/dump.sql'
+      - docker-volume-backup.exec-post=/bin/sh -c 'echo "post" > /tmp/volume/post.txt'
+      - docker-volume-backup.exec-label=test
+    volumes:
+      - app_data:/tmp/volume
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      EXEC_LABEL: test
+      EXEC_FORWARD_OUTPUT: "true"
+    volumes:
+      - archive:/archive
+      - app_data:/backup/data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  app_data:
+  archive:

test/commands/run.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+
+
+docker-compose up -d
+sleep 30 # mariadb likes to take a bit before responding
+
+docker-compose exec backup backup
+sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' commands_archive) ./local
+
+tar -xvf ./local/test.tar.gz
+if [ ! -f ./backup/data/dump.sql ]; then
+  echo "[TEST:FAIL] Could not find file written by pre command."
+  exit 1
+fi
+echo "[TEST:PASS] Found expected file."
+
+if [ -f ./backup/data/post.txt ]; then
+  echo "[TEST:FAIL] File created in post command was present in backup."
+  exit 1
+fi
+echo "[TEST:PASS] Did not find unexpected file."
+
+docker-compose down --volumes
+sudo rm -rf ./local
+
+
+echo "[TEST:INFO] Running commands test in swarm mode next."
+
+docker swarm init
+
+docker stack deploy --compose-file=docker-compose.yml test_stack
+
+while [ -z $(docker ps -q -f name=backup) ]; do
+  echo "[TEST:INFO] Backup container not ready yet. Retrying."
+  sleep 1
+done
+
+sleep 20
+
+docker exec $(docker ps -q -f name=backup) backup
+
+sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' test_stack_archive) ./local
+
+tar -xvf ./local/test.tar.gz
+if [ ! -f ./backup/data/dump.sql ]; then
+  echo "[TEST:FAIL] Could not find file written by pre command."
+  exit 1
+fi
+echo "[TEST:PASS] Found expected file."
+
+if [ -f ./backup/data/post.txt ]; then
+  echo "[TEST:FAIL] File created in post command was present in backup."
+  exit 1
+fi
+echo "[TEST:PASS] Did not find unexpected file."
+
+docker stack rm test_stack
+docker swarm leave --force

test/compose/docker-compose.yml

@@ -10,12 +10,23 @@ services:
       MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
     entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
     volumes:
-      - backup_data:/data
+      - minio_backup_data:/data
 
-  backup: &default_backup_service
-    image: offen/docker-volume-backup:${TEST_VERSION}
+  webdav:
+    image: bytemark/webdav:2.4
+    environment:
+      AUTH_TYPE: Digest
+      USERNAME: test
+      PASSWORD: test
+    volumes:
+      - webdav_backup_data:/var/lib/dav
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
     depends_on:
       - minio
+      - webdav
     restart: always
     environment:
       AWS_ACCESS_KEY_ID: test
@@ -23,8 +34,18 @@ services:
       AWS_ENDPOINT: minio:9000
       AWS_ENDPOINT_PROTO: http
       AWS_S3_BUCKET_NAME: backup
-      BACKUP_FILENAME: test.tar.gz
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      GPG_PASSPHRASE: 1234secret
+      WEBDAV_URL: http://webdav/
+      WEBDAV_PATH: /my/new/path/
+      WEBDAV_USERNAME: test
+      WEBDAV_PASSWORD: test
     volumes:
       - ./local:/archive
       - app_data:/backup/app_data:ro
@@ -37,7 +58,7 @@ services:
     volumes:
       - app_data:/var/opt/offen
 
-
 volumes:
-  backup_data:
+  minio_backup_data:
+  webdav_backup_data:
   app_data:

test/compose/run.sh

@@ -9,24 +9,60 @@ mkdir -p local
 docker-compose up -d
 sleep 5
 
+# A symlink for a known file in the volume is created so the test can check
+# whether symlinks are preserved on backup.
+docker-compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link
 docker-compose exec backup backup
 
-docker run --rm -it \
-  -v compose_backup_data:/data alpine \
-  ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
-
-echo "[TEST:PASS] Found relevant files in untared remote backup."
-
-tar -xf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
-
-echo "[TEST:PASS] Found relevant files in untared local backup."
-
-if [ "$(docker-compose ps -q | wc -l)" != "3" ]; then
+sleep 5
+if [ "$(docker-compose ps -q | wc -l)" != "4" ]; then
   echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
   docker-compose ps
   exit 1
 fi
-
 echo "[TEST:PASS] All containers running post backup."
 
+
+docker run --rm -it \
+  -v compose_minio_backup_data:/minio_data \
+  -v compose_webdav_backup_data:/webdav_data alpine \
+  ash -c 'apk add gnupg && \
+          echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /minio_data/backup/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xvf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db && \
+          echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /webdav_data/data/my/new/path/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xvf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+echo "[TEST:PASS] Found relevant files in decrypted and untared remote backups."
+
+echo 1234secret | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 ./local/test-hostnametoken.tar.gz.gpg > ./local/decrypted.tar.gz
+tar -xf ./local/decrypted.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
+rm ./local/decrypted.tar.gz
+test -L /tmp/backup/app_data/db.link
+
+echo "[TEST:PASS] Found relevant files in decrypted and untared local backup."
+
+test -L ./local/test-hostnametoken.latest.tar.gz.gpg
+echo "[TEST:PASS] Found symlink to latest version in local backup."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker-compose up -d
+sleep 5
+
+docker-compose exec backup backup
+
+docker run --rm -it \
+  -v compose_minio_backup_data:/minio_data \
+  -v compose_webdav_backup_data:/webdav_data alpine \
+  ash -c '[ $(find /minio_data/backup/ -type f | wc -l) = "1" ] && \
+          [ $(find /webdav_data/data/my/new/path/ -type f | wc -l) = "1" ]'
+
+echo "[TEST:PASS] Remote backups have not been deleted."
+
+if [ "$(find ./local -type f | wc -l)" != "1" ]; then
+  echo "[TEST:FAIL] Backups should not have been deleted, instead seen:"
+  find ./local -type f
+  exit 1
+fi
+echo "[TEST:PASS] Local backups have not been deleted."
+
 docker-compose down --volumes

test/confd/.gitignore

@@ -0,0 +1 @@
+local

test/confd/01backup.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="conf.tar.gz"
+BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/02backup.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="other.tar.gz"
+BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/03never.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="never.tar.gz"
+BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?"

test/confd/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    volumes:
+      - ./local:/archive
+      - app_data:/backup/app_data:ro
+      - ./01backup.env:/etc/dockervolumebackup/conf.d/01backup.env
+      - ./02backup.env:/etc/dockervolumebackup/conf.d/02backup.env
+      - ./03never.env:/etc/dockervolumebackup/conf.d/03never.env
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/confd/run.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+
+mkdir -p local
+
+docker-compose up -d
+
+# sleep until a backup is guaranteed to have happened on the 1 minute schedule
+sleep 100
+
+docker-compose down --volumes
+
+if [ ! -f ./local/conf.tar.gz ]; then
+  echo "[TEST:FAIL] Config from file was not used."
+  exit 1
+fi
+echo "[TEST:PASS] Config from file was used."
+
+if [ ! -f ./local/other.tar.gz ]; then
+  echo "[TEST:FAIL] Run on same schedule did not succeed."
+  exit 1
+fi
+echo "[TEST:PASS] Run on same schedule succeeded."
+
+if [ -f ./local/never.tar.gz ]; then
+  echo "[TEST:FAIL] Unexpected file was found."
+  exit 1
+fi
+echo "[TEST:PASS] Unexpected cron did not run."

test/notifications/.gitignore

@@ -0,0 +1 @@
+local

test/notifications/docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_PRUNING_PREFIX: test
+      NOTIFICATION_LEVEL: info
+      NOTIFICATION_URLS: ${NOTIFICATION_URLS}
+    volumes:
+      - ./local:/archive
+      - app_data:/backup/app_data:ro
+      - ./notifications.tmpl:/etc/dockervolumebackup/notifications.d/notifications.tmpl
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+  gotify:
+    image: gotify/server
+    ports:
+      - 8080:80
+    environment:
+      - GOTIFY_DEFAULTUSER_PASS=custom
+    volumes:
+      - gotify_data:/app/data
+
+volumes:
+  app_data:
+  gotify_data:

test/notifications/notifications.tmpl

@@ -0,0 +1,7 @@
+{{ define "title_success" -}}
+Successful test run, yay!
+{{- end }}
+
+{{ define "body_success" -}}
+Backing up {{ .Stats.BackupFile.FullPath }} succeeded.
+{{- end }}

test/notifications/run.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+
+mkdir -p local
+
+docker-compose up -d
+sleep 5
+
+GOTIFY_TOKEN=$(curl -sSLX POST -H 'Content-Type: application/json' -d '{"name":"test"}' http://admin:custom@localhost:8080/application | jq -r '.token')
+echo "[TEST:INFO] Set up Gotify application using token $GOTIFY_TOKEN"
+
+docker-compose exec backup backup
+
+NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
+if [ "$NUM_MESSAGES" != 0 ]; then
+  echo "[TEST:FAIL] Expected no notifications to be sent when not configured"
+  exit 1
+fi
+echo "[TEST:PASS] No notifications were sent when not configured."
+
+docker-compose down
+
+NOTIFICATION_URLS="gotify://gotify/${GOTIFY_TOKEN}?disableTLS=true" docker-compose up -d
+
+docker-compose exec backup backup
+
+NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
+if [ "$NUM_MESSAGES" != 1 ]; then
+  echo "[TEST:FAIL] Expected one notifications to be sent when configured"
+  exit 1
+fi
+echo "[TEST:PASS] Correct number of notifications were sent when configured."
+
+MESSAGE_TITLE=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].title')
+MESSAGE_BODY=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].message')
+
+if [ "$MESSAGE_TITLE" != "Successful test run, yay!" ]; then
+  echo "[TEST:FAIL] Unexpected notification title $MESSAGE_TITLE"
+  exit 1
+fi
+echo "[TEST:PASS] Custom notification title was used."
+
+if [ "$MESSAGE_BODY" != "Backing up /tmp/test.tar.gz succeeded." ]; then
+  echo "[TEST:FAIL] Unexpected notification body $MESSAGE_BODY"
+  exit 1
+fi
+echo "[TEST:PASS] Custom notification body was used."
+
+docker-compose down --volumes

test/ownership/.gitignore

@@ -0,0 +1 @@
+local

test/ownership/docker-compose.yml

@@ -0,0 +1,27 @@
+version: '3'
+
+services:
+  db:
+    image: postgres:14-alpine
+    restart: unless-stopped
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=1FHJMSwt0yhIN1zS7I4DilGUhThBKq0x
+      - POSTGRES_USER=test
+      - POSTGRES_DB=test
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION}
+    restart: always
+    environment:
+      BACKUP_FILENAME: backup.tar.gz
+    volumes:
+      - postgres_data:/backup/postgres:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./local:/archive
+
+volumes:
+  postgres_data:

test/ownership/run.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+# This test refers to https://github.com/offen/docker-volume-backup/issues/71
+
+set -e
+
+cd $(dirname $0)
+
+mkdir -p local
+
+docker-compose up -d
+sleep 5
+
+docker-compose exec backup backup
+
+sudo tar --same-owner -xvf ./local/backup.tar.gz -C /tmp
+
+sudo find /tmp/backup/postgres > /dev/null
+echo "[TEST:PASS] Backup contains files at expected location"
+
+for file in $(sudo find /tmp/backup/postgres); do
+  if [ "$(sudo stat -c '%u:%g' $file)" != "70:70" ]; then
+    echo "[TEST:FAIL] Unexpected file ownership for $file: $(sudo stat -c '%u:%g' $file)"
+    exit 1
+  fi
+done
+echo "[TEST:PASS] All files and directories in backup preserved their ownership."
+
+docker-compose down --volumes

test/swarm/docker-compose.yml

@@ -18,8 +18,8 @@ services:
     volumes:
       - backup_data:/data
 
-  backup: &default_backup_service
-    image: offen/docker-volume-backup:${TEST_VERSION}
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
     depends_on:
       - minio
     deploy:
@@ -33,6 +33,8 @@ services:
       AWS_S3_BUCKET_NAME: backup
       BACKUP_FILENAME: test.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: 7
+      BACKUP_PRUNING_LEEWAY: 5s
     volumes:
       - pg_data:/backup/pg_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
@@ -41,13 +43,15 @@ services:
     image: offen/offen:latest
     labels:
       - docker-volume-backup.stop-during-backup=true
+    healthcheck:
+      disable: true
     deploy:
       replicas: 2
       restart_policy:
         condition: on-failure
 
   pg:
-    image: postgres:12.2-alpine
+    image: postgres:14-alpine
     environment:
       POSTGRES_PASSWORD: example
     labels:

test/swarm/run.sh

@@ -23,14 +23,13 @@ docker run --rm -it \
 
 echo "[TEST:PASS] Found relevant files in untared backup."
 
+sleep 5
 if [ "$(docker ps -q | wc -l)" != "5" ]; then
   echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
   docker ps -a
   exit 1
 fi
-
 echo "[TEST:PASS] All containers running post backup."
 
 docker stack rm test_stack
-
 docker swarm leave --force