shorten log messages

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 jobs:
   canary:
     machine:
-      image: ubuntu-2004:202201-02
+      image: ubuntu-1604:202007-01
     working_directory: ~/docker-volume-backup
     steps:
       - checkout
@@ -11,15 +11,10 @@ jobs:
           name: Build
           command: |
             docker build . -t offen/docker-volume-backup:canary
-      - run:
-          name: Install gnupg
-          command: |
-            sudo apt-get install -y gnupg
       - run:
           name: Run tests
           working_directory: ~/docker-volume-backup/test
           command: |
-            export GPG_TTY=$(tty)
             ./test.sh canary
 
   build:
@@ -48,7 +43,6 @@ jobs:
             if [[ "$CIRCLE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
               # prerelease tags like `v2.0.0-alpha.1` should not be released as `latest`
               tag_args="$tag_args -t offen/docker-volume-backup:latest"
-              tag_args="$tag_args -t offen/docker-volume-backup:$(echo "$CIRCLE_TAG" | cut -d. -f1)"
             fi
             docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 \
                $tag_args . --push

.github/ISSUE_TEMPLATE.md

@@ -1,20 +0,0 @@
-* **I'm submitting a ...**
-  - [ ] bug report
-  - [ ] feature request
-  - [ ] support request
-
-* **What is the current behavior?**
-
-* **If the current behavior is a bug, please provide the configuration and steps to reproduce and if possible a minimal demo of the problem.**
-
-* **What is the expected behavior?**
-
-* **What is the motivation / use case for changing the behavior?**
-
-* **Please tell us about your environment:**
-  
-  - Image version:
-  - Docker version: 
-  - docker-compose version:
-
-* **Other information** (e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, eg. stackoverflow, etc)

Dockerfile

@@ -1,22 +1,20 @@
 # Copyright 2021 - Offen Authors <hioffen@posteo.de>
 # SPDX-License-Identifier: MPL-2.0
 
-FROM golang:1.18-alpine as builder
+FROM golang:1.17-alpine as builder
 
 WORKDIR /app
 COPY go.mod go.sum ./
-RUN go mod download
+COPY cmd/backup/main.go ./cmd/backup/main.go
-COPY cmd/backup ./cmd/backup/
+RUN go build -o backup cmd/backup/main.go
-WORKDIR /app/cmd/backup
-RUN go build -o backup .
 
-FROM alpine:3.15
+FROM alpine:3.14
 
 WORKDIR /root
 
-RUN apk add --no-cache ca-certificates
+RUN apk add --update ca-certificates
 
-COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
+COPY --from=builder /app/backup /usr/bin/backup
 
 COPY ./entrypoint.sh /root/
 RUN chmod +x entrypoint.sh

NOTICE

@@ -0,0 +1,22 @@
+Copyright 2021 Offen Authors <hioffen@posteo.de>
+
+This project contains software that is Copyright (c) 2018 Futurice
+Licensed under the Terms of the MIT License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cmd/backup/archive.go

@@ -1,133 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-// Portions of this file are taken from package `targz`, Copyright (c) 2014 Fredrik Wallgren
-// Licensed under the MIT License: https://github.com/walle/targz/blob/57fe4206da5abf7dd3901b4af3891ec2f08c7b08/LICENSE
-
-package main
-
-import (
-	"archive/tar"
-	"compress/gzip"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-)
-
-func createArchive(files []string, inputFilePath, outputFilePath string) error {
-	inputFilePath = stripTrailingSlashes(inputFilePath)
-	inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
-	if err != nil {
-		return fmt.Errorf("createArchive: error transposing given file paths: %w", err)
-	}
-	if err := os.MkdirAll(filepath.Dir(outputFilePath), 0755); err != nil {
-		return fmt.Errorf("createArchive: error creating output file path: %w", err)
-	}
-
-	if err := compress(files, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
-		return fmt.Errorf("createArchive: error creating archive: %w", err)
-	}
-
-	return nil
-}
-
-func stripTrailingSlashes(path string) string {
-	if len(path) > 0 && path[len(path)-1] == '/' {
-		path = path[0 : len(path)-1]
-	}
-
-	return path
-}
-
-func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error) {
-	inputFilePath, err := filepath.Abs(inputFilePath)
-	if err == nil {
-		outputFilePath, err = filepath.Abs(outputFilePath)
-	}
-
-	return inputFilePath, outputFilePath, err
-}
-
-func compress(paths []string, outFilePath, subPath string) error {
-	file, err := os.Create(outFilePath)
-	if err != nil {
-		return fmt.Errorf("compress: error creating out file: %w", err)
-	}
-
-	prefix := path.Dir(outFilePath)
-	gzipWriter := gzip.NewWriter(file)
-	tarWriter := tar.NewWriter(gzipWriter)
-
-	for _, p := range paths {
-		if err := writeTarGz(p, tarWriter, prefix); err != nil {
-			return fmt.Errorf("compress error writing %s to archive: %w", p, err)
-		}
-	}
-
-	err = tarWriter.Close()
-	if err != nil {
-		return fmt.Errorf("compress: error closing tar writer: %w", err)
-	}
-
-	err = gzipWriter.Close()
-	if err != nil {
-		return fmt.Errorf("compress: error closing gzip writer: %w", err)
-	}
-
-	err = file.Close()
-	if err != nil {
-		return fmt.Errorf("compress: error closing file: %w", err)
-	}
-
-	return nil
-}
-
-func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
-	fileInfo, err := os.Lstat(path)
-	if err != nil {
-		return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err)
-	}
-
-	if fileInfo.Mode()&os.ModeSocket == os.ModeSocket {
-		return nil
-	}
-
-	var link string
-	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
-		var err error
-		if link, err = os.Readlink(path); err != nil {
-			return fmt.Errorf("writeTarGz: error resolving symlink %s: %w", path, err)
-		}
-	}
-
-	header, err := tar.FileInfoHeader(fileInfo, link)
-	if err != nil {
-		return fmt.Errorf("writeTarGz: error getting file info header: %w", err)
-	}
-	header.Name = strings.TrimPrefix(path, prefix)
-
-	err = tarWriter.WriteHeader(header)
-	if err != nil {
-		return fmt.Errorf("writeTarGz: error writing file info header: %w", err)
-	}
-
-	if !fileInfo.Mode().IsRegular() {
-		return nil
-	}
-
-	file, err := os.Open(path)
-	if err != nil {
-		return fmt.Errorf("writeTarGz: error opening %s: %w", path, err)
-	}
-	defer file.Close()
-
-	_, err = io.Copy(tarWriter, file)
-	if err != nil {
-		return fmt.Errorf("writeTarGz: error copying %s to tar writer: %w", path, err)
-	}
-
-	return nil
-}

cmd/backup/config.go

@@ -1,67 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-package main
-
-import (
-	"fmt"
-	"regexp"
-	"time"
-)
-
-// Config holds all configuration values that are expected to be set
-// by users.
-type Config struct {
-	BackupSources              string        `split_words:"true" default:"/backup"`
-	BackupFilename             string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
-	BackupFilenameExpand       bool          `split_words:"true"`
-	BackupLatestSymlink        string        `split_words:"true"`
-	BackupArchive              string        `split_words:"true" default:"/archive"`
-	BackupRetentionDays        int32         `split_words:"true" default:"-1"`
-	BackupPruningLeeway        time.Duration `split_words:"true" default:"1m"`
-	BackupPruningPrefix        string        `split_words:"true"`
-	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
-	BackupFromSnapshot         bool          `split_words:"true"`
-	BackupExcludeRegexp        RegexpDecoder `split_words:"true"`
-	AwsS3BucketName            string        `split_words:"true"`
-	AwsS3Path                  string        `split_words:"true"`
-	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
-	AwsEndpointProto           string        `split_words:"true" default:"https"`
-	AwsEndpointInsecure        bool          `split_words:"true"`
-	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
-	AwsSecretAccessKey         string        `split_words:"true"`
-	AwsIamRoleEndpoint         string        `split_words:"true"`
-	GpgPassphrase              string        `split_words:"true"`
-	NotificationURLs           []string      `envconfig:"NOTIFICATION_URLS"`
-	NotificationLevel          string        `split_words:"true" default:"error"`
-	EmailNotificationRecipient string        `split_words:"true"`
-	EmailNotificationSender    string        `split_words:"true" default:"noreply@nohost"`
-	EmailSMTPHost              string        `envconfig:"EMAIL_SMTP_HOST"`
-	EmailSMTPPort              int           `envconfig:"EMAIL_SMTP_PORT" default:"587"`
-	EmailSMTPUsername          string        `envconfig:"EMAIL_SMTP_USERNAME"`
-	EmailSMTPPassword          string        `envconfig:"EMAIL_SMTP_PASSWORD"`
-	WebdavUrl                  string        `split_words:"true"`
-	WebdavUrlInsecure          bool          `split_words:"true"`
-	WebdavPath                 string        `split_words:"true" default:"/"`
-	WebdavUsername             string        `split_words:"true"`
-	WebdavPassword             string        `split_words:"true"`
-	ExecLabel                  string        `split_words:"true"`
-	ExecForwardOutput          bool          `split_words:"true"`
-	LockTimeout                time.Duration `split_words:"true" default:"60m"`
-}
-
-type RegexpDecoder struct {
-	Re *regexp.Regexp
-}
-
-func (r *RegexpDecoder) Decode(v string) error {
-	if v == "" {
-		return nil
-	}
-	re, err := regexp.Compile(v)
-	if err != nil {
-		return fmt.Errorf("config: error compiling given regexp `%s`: %w", v, err)
-	}
-	*r = RegexpDecoder{Re: re}
-	return nil
-}

cmd/backup/exec.go

@@ -1,123 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-// Portions of this file are taken and adapted from `moby`, Copyright 2012-2017 Docker, Inc.
-// Licensed under the Apache 2.0 License: https://github.com/moby/moby/blob/8e610b2b55bfd1bfa9436ab110d311f5e8a74dcb/LICENSE
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-
-	"github.com/cosiner/argv"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/pkg/stdcopy"
-	"golang.org/x/sync/errgroup"
-)
-
-func (s *script) exec(containerRef string, command string) ([]byte, []byte, error) {
-	args, _ := argv.Argv(command, nil, nil)
-	execID, err := s.cli.ContainerExecCreate(context.Background(), containerRef, types.ExecConfig{
-		Cmd:          args[0],
-		AttachStdin:  true,
-		AttachStderr: true,
-	})
-	if err != nil {
-		return nil, nil, fmt.Errorf("exec: error creating container exec: %w", err)
-	}
-
-	resp, err := s.cli.ContainerExecAttach(context.Background(), execID.ID, types.ExecStartCheck{})
-	if err != nil {
-		return nil, nil, fmt.Errorf("exec: error attaching container exec: %w", err)
-	}
-	defer resp.Close()
-
-	var outBuf, errBuf bytes.Buffer
-	outputDone := make(chan error)
-
-	go func() {
-		_, err := stdcopy.StdCopy(&outBuf, &errBuf, resp.Reader)
-		outputDone <- err
-	}()
-
-	select {
-	case err := <-outputDone:
-		if err != nil {
-			return nil, nil, fmt.Errorf("exec: error demultiplexing output: %w", err)
-		}
-		break
-	}
-
-	stdout, err := ioutil.ReadAll(&outBuf)
-	if err != nil {
-		return nil, nil, fmt.Errorf("exec: error reading stdout: %w", err)
-	}
-	stderr, err := ioutil.ReadAll(&errBuf)
-	if err != nil {
-		return nil, nil, fmt.Errorf("exec: error reading stderr: %w", err)
-	}
-
-	res, err := s.cli.ContainerExecInspect(context.Background(), execID.ID)
-	if err != nil {
-		return nil, nil, fmt.Errorf("exec: error inspecting container exec: %w", err)
-	}
-
-	if res.ExitCode > 0 {
-		return stdout, stderr, fmt.Errorf("exec: running command exited %d", res.ExitCode)
-	}
-
-	return stdout, stderr, nil
-}
-
-func (s *script) runLabeledCommands(label string) error {
-	f := []filters.KeyValuePair{
-		{Key: "label", Value: label},
-	}
-	if s.c.ExecLabel != "" {
-		f = append(f, filters.KeyValuePair{
-			Key:   "label",
-			Value: fmt.Sprintf("docker-volume-backup.exec-label=%s", s.c.ExecLabel),
-		})
-	}
-	containersWithCommand, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Quiet:   true,
-		Filters: filters.NewArgs(f...),
-	})
-	if err != nil {
-		return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
-	}
-
-	if len(containersWithCommand) == 0 {
-		return nil
-	}
-
-	g := new(errgroup.Group)
-
-	for _, container := range containersWithCommand {
-		c := container
-		g.Go(func() error {
-			cmd, _ := c.Labels[label]
-			s.logger.Infof("Running %s command %s for container %s", label, cmd, strings.TrimPrefix(c.Names[0], "/"))
-			stdout, stderr, err := s.exec(c.ID, cmd)
-			if s.c.ExecForwardOutput {
-				os.Stderr.Write(stderr)
-				os.Stdout.Write(stdout)
-			}
-			if err != nil {
-				return fmt.Errorf("runLabeledCommands: error executing command: %w", err)
-			}
-			return nil
-		})
-	}
-
-	if err := g.Wait(); err != nil {
-		return fmt.Errorf("runLabeledCommands: error from errgroup: %w", err)
-	}
-	return nil
-}

cmd/backup/hooks.go

@@ -1,56 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-package main
-
-import (
-	"fmt"
-	"sort"
-)
-
-// hook contains a queued action that can be trigger them when the script
-// reaches a certain point (e.g. unsuccessful backup)
-type hook struct {
-	level  hookLevel
-	action func(err error) error
-}
-
-type hookLevel int
-
-const (
-	hookLevelPlumbing hookLevel = iota
-	hookLevelError
-	hookLevelInfo
-)
-
-var hookLevels = map[string]hookLevel{
-	"info":  hookLevelInfo,
-	"error": hookLevelError,
-}
-
-// registerHook adds the given action at the given level.
-func (s *script) registerHook(level hookLevel, action func(err error) error) {
-	s.hooks = append(s.hooks, hook{level, action})
-}
-
-// runHooks runs all hooks that have been registered using the
-// given levels in the defined ordering. In case executing a hook returns an
-// error, the following hooks will still be run before the function returns.
-func (s *script) runHooks(err error) error {
-	sort.SliceStable(s.hooks, func(i, j int) bool {
-		return s.hooks[i].level < s.hooks[j].level
-	})
-	var actionErrors []error
-	for _, hook := range s.hooks {
-		if hook.level > s.hookLevel {
-			continue
-		}
-		if actionErr := hook.action(err); actionErr != nil {
-			actionErrors = append(actionErrors, fmt.Errorf("runHooks: error running hook: %w", actionErr))
-		}
-	}
-	if len(actionErrors) != 0 {
-		return join(actionErrors...)
-	}
-	return nil
-}

cmd/backup/lock.go

@@ -1,58 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-package main
-
-import (
-	"errors"
-	"fmt"
-	"time"
-
-	"github.com/gofrs/flock"
-)
-
-// lock opens a lockfile at the given location, keeping it locked until the
-// caller invokes the returned release func. In case the lock is currently blocked
-// by another execution, it will repeatedly retry until the lock is available
-// or the given timeout is exceeded.
-func (s *script) lock(lockfile string) (func() error, error) {
-	start := time.Now()
-	defer func() {
-		s.stats.LockedTime = time.Now().Sub(start)
-	}()
-
-	retry := time.NewTicker(5 * time.Second)
-	defer retry.Stop()
-	deadline := time.NewTimer(s.c.LockTimeout)
-	defer deadline.Stop()
-
-	fileLock := flock.New(lockfile)
-
-	for {
-		acquired, err := fileLock.TryLock()
-		if err != nil {
-			return noop, fmt.Errorf("lock: error trying lock: %w", err)
-		}
-		if acquired {
-			if s.encounteredLock {
-				s.logger.Info("Acquired exclusive lock on subsequent attempt, ready to continue.")
-			}
-			return fileLock.Unlock, nil
-		}
-
-		if !s.encounteredLock {
-			s.logger.Infof(
-				"Exclusive lock was not available on first attempt. Will retry until it becomes available or the timeout of %s is exceeded.",
-				s.c.LockTimeout,
-			)
-			s.encounteredLock = true
-		}
-
-		select {
-		case <-retry.C:
-			continue
-		case <-deadline.C:
-			return noop, errors.New("lock: timed out waiting for lockfile to become available")
-		}
-	}
-}

cmd/backup/main.go

@@ -1,65 +1,508 @@
-// Copyright 2021-2022 - Offen Authors <hioffen@posteo.de>
+// Copyright 2021 - Offen Authors <hioffen@posteo.de>
 // SPDX-License-Identifier: MPL-2.0
 
 package main
 
 import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+	"github.com/joho/godotenv"
+	"github.com/leekchan/timeutil"
+	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/sirupsen/logrus"
+	"github.com/walle/targz"
+	"golang.org/x/crypto/openpgp"
 )
 
 func main() {
-	s, err := newScript()
+	unlock := lock("/var/dockervolumebackup.lock")
+	defer unlock()
+
+	s := &script{}
+	s.must(s.init())
+	s.must(s.stopContainersAndRun(s.takeBackup))
+	s.must(s.encryptBackup())
+	s.must(s.copyBackup())
+	s.must(s.cleanBackup())
+	s.must(s.pruneOldBackups())
+	s.logger.Info("Finished running backup tasks.")
+}
+
+type script struct {
+	ctx    context.Context
+	cli    *client.Client
+	mc     *minio.Client
+	logger *logrus.Logger
+
+	start time.Time
+
+	file           string
+	bucket         string
+	archive        string
+	sources        string
+	passphrase     []byte
+	retentionDays  *int
+	leeway         *time.Duration
+	containerLabel string
+	pruningPrefix  string
+}
+
+// lock opens a lockfile at the given location, keeping it locked until the
+// caller invokes the returned release func. When invoked while the file is
+// still locked the function panics.
+func lock(lockfile string) func() error {
+	lf, err := os.OpenFile(lockfile, os.O_CREATE, os.ModeAppend)
 	if err != nil {
 		panic(err)
 	}
+	return func() error {
+		if err := lf.Close(); err != nil {
+			return fmt.Errorf("lock: error releasing file lock: %w", err)
+		}
+		if err := os.Remove(lockfile); err != nil {
+			return fmt.Errorf("lock: error removing lock file: %w", err)
+		}
+		return nil
+	}
+}
 
-	unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
+// init creates all resources needed for the script to perform actions against
-	defer unlock()
+// remote resources like the Docker engine or remote storage locations.
-	s.must(err)
+func (s *script) init() error {
+	s.ctx = context.Background()
+	s.logger = logrus.New()
+	s.logger.SetOutput(os.Stdout)
 
-	defer func() {
+	if err := godotenv.Load("/etc/backup.env"); err != nil {
-		if pArg := recover(); pArg != nil {
+		return fmt.Errorf("init: failed to load env file: %w", err)
-			if err, ok := pArg.(error); ok {
+	}
-				if hookErr := s.runHooks(err); hookErr != nil {
+
-					s.logger.Errorf("An error occurred calling the registered hooks: %s", hookErr)
+	_, err := os.Stat("/var/run/docker.sock")
-				}
+	if !os.IsNotExist(err) {
-				os.Exit(1)
+		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+		if err != nil {
+			return fmt.Errorf("init: failed to create docker client")
+		}
+		s.cli = cli
+	}
+
+	if bucket := os.Getenv("AWS_S3_BUCKET_NAME"); bucket != "" {
+		s.bucket = bucket
+		mc, err := minio.New(os.Getenv("AWS_ENDPOINT"), &minio.Options{
+			Creds: credentials.NewStaticV4(
+				os.Getenv("AWS_ACCESS_KEY_ID"),
+				os.Getenv("AWS_SECRET_ACCESS_KEY"),
+				"",
+			),
+			Secure: os.Getenv("AWS_ENDPOINT_INSECURE") == "" && os.Getenv("AWS_ENDPOINT_PROTO") == "https",
+		})
+		if err != nil {
+			return fmt.Errorf("init: error setting up minio client: %w", err)
+		}
+		s.mc = mc
+	}
+
+	file := os.Getenv("BACKUP_FILENAME")
+	if file == "" {
+		return errors.New("init: BACKUP_FILENAME not given")
+	}
+	s.file = path.Join("/tmp", file)
+	s.archive = os.Getenv("BACKUP_ARCHIVE")
+	s.sources = os.Getenv("BACKUP_SOURCES")
+	if v := os.Getenv("GPG_PASSPHRASE"); v != "" {
+		s.passphrase = []byte(v)
+	}
+	if v := os.Getenv("BACKUP_RETENTION_DAYS"); v != "" {
+		i, err := strconv.Atoi(v)
+		if err != nil {
+			return fmt.Errorf("init: error parsing BACKUP_RETENTION_DAYS as int: %w", err)
+		}
+		s.retentionDays = &i
+	}
+	if v := os.Getenv("BACKUP_PRUNING_LEEWAY"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			return fmt.Errorf("init: error parsing BACKUP_PRUNING_LEEWAY as duration: %w", err)
+		}
+		s.leeway = &d
+	}
+	s.containerLabel = os.Getenv("BACKUP_STOP_CONTAINER_LABEL")
+	s.pruningPrefix = os.Getenv("BACKUP_PRUNING_PREFIX")
+	s.start = time.Now()
+
+	return nil
+}
+
+// stopContainersAndRun stops all Docker containers that are marked as to being
+// stopped during the backup and runs the given thunk. After returning, it makes
+// sure containers are being restarted if required.
+func (s *script) stopContainersAndRun(thunk func() error) error {
+	if s.cli == nil {
+		return thunk()
+	}
+
+	allContainers, err := s.cli.ContainerList(s.ctx, types.ContainerListOptions{
+		Quiet: true,
+	})
+	if err != nil {
+		return fmt.Errorf("stopContainersAndRun: error querying for containers: %w", err)
+	}
+
+	containerLabel := fmt.Sprintf(
+		"docker-volume-backup.stop-during-backup=%s",
+		s.containerLabel,
+	)
+	containersToStop, err := s.cli.ContainerList(s.ctx, types.ContainerListOptions{
+		Quiet: true,
+		Filters: filters.NewArgs(filters.KeyValuePair{
+			Key:   "label",
+			Value: containerLabel,
+		}),
+	})
+
+	if err != nil {
+		return fmt.Errorf("stopContainersAndRun: error querying for containers to stop: %w", err)
+	}
+	s.logger.Infof(
+		"Stopping %d containers labeled `%s` out of %d running container(s).",
+		len(containersToStop),
+		containerLabel,
+		len(allContainers),
+	)
+
+	var stoppedContainers []types.Container
+	var errors []error
+	if len(containersToStop) != 0 {
+		for _, container := range containersToStop {
+			if err := s.cli.ContainerStop(s.ctx, container.ID, nil); err != nil {
+				errors = append(errors, err)
+			} else {
+				stoppedContainers = append(stoppedContainers, container)
+			}
+		}
+	}
+
+	defer func() error {
+		servicesRequiringUpdate := map[string]struct{}{}
+
+		var restartErrors []error
+		for _, container := range stoppedContainers {
+			if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok {
+				servicesRequiringUpdate[swarmServiceName] = struct{}{}
+				continue
+			}
+			if err := s.cli.ContainerStart(s.ctx, container.ID, types.ContainerStartOptions{}); err != nil {
+				restartErrors = append(restartErrors, err)
 			}
-			panic(pArg)
 		}
 
-		if err := s.runHooks(nil); err != nil {
+		if len(servicesRequiringUpdate) != 0 {
-			s.logger.Errorf(
+			services, _ := s.cli.ServiceList(s.ctx, types.ServiceListOptions{})
-				"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
+			for serviceName := range servicesRequiringUpdate {
+				var serviceMatch swarm.Service
+				for _, service := range services {
+					if service.Spec.Name == serviceName {
+						serviceMatch = service
+						break
+					}
+				}
+				if serviceMatch.ID == "" {
+					return fmt.Errorf("stopContainersAndRun: Couldn't find service with name %s", serviceName)
+				}
+				serviceMatch.Spec.TaskTemplate.ForceUpdate = 1
+				_, err := s.cli.ServiceUpdate(
+					s.ctx, serviceMatch.ID,
+					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},
+				)
+				if err != nil {
+					restartErrors = append(restartErrors, err)
+				}
+			}
+		}
+
+		if len(restartErrors) != 0 {
+			return fmt.Errorf(
+				"stopContainersAndRun: %d error(s) restarting containers and services: %w",
+				len(restartErrors),
 				err,
 			)
-			os.Exit(1)
 		}
-		s.logger.Info("Finished running backup tasks.")
+		s.logger.Infof("Restarted %d container(s) and the matching service(s).", len(stoppedContainers))
+		return nil
 	}()
 
-	s.must(func() error {
+	var stopErr error
-		runPostCommands, err := s.runCommands()
+	if len(errors) != 0 {
-		defer func() {
+		stopErr = fmt.Errorf(
-			s.must(runPostCommands())
+			"stopContainersAndRun: %d errors stopping containers: %w",
-		}()
+			len(errors),
-		if err != nil {
+			err,
-			return err
+		)
-		}
+	}
-		restartContainers, err := s.stopContainers()
+	if stopErr != nil {
-		// The mechanism for restarting containers is not using hooks as it
+		return stopErr
-		// should happen as soon as possible (i.e. before uploading backups or
+	}
-		// similar).
-		defer func() {
-			s.must(restartContainers())
-		}()
-		if err != nil {
-			return err
-		}
-		return s.takeBackup()
-	}())
 
-	s.must(s.encryptBackup())
+	return thunk()
-	s.must(s.copyBackup())
+}
-	s.must(s.pruneBackups())
+
+// takeBackup creates a tar archive of the configured backup location and
+// saves it to disk.
+func (s *script) takeBackup() error {
+	s.file = timeutil.Strftime(&s.start, s.file)
+	if err := targz.Compress(s.sources, s.file); err != nil {
+		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
+	}
+	s.logger.Infof("Created backup of `%s` at `%s`.", s.sources, s.file)
+	return nil
+}
+
+// encryptBackup encrypts the backup file using PGP and the configured passphrase.
+// In case no passphrase is given it returns early, leaving the backup file
+//  untouched.
+func (s *script) encryptBackup() error {
+	if s.passphrase == nil {
+		return nil
+	}
+
+	buf := bytes.NewBuffer(nil)
+	_, name := path.Split(s.file)
+	pt, err := openpgp.SymmetricallyEncrypt(buf, []byte(s.passphrase), &openpgp.FileHints{
+		IsBinary: true,
+		FileName: name,
+	}, nil)
+	if err != nil {
+		return fmt.Errorf("encryptBackup: error encrypting backup file: %w", err)
+	}
+
+	unencrypted, err := ioutil.ReadFile(s.file)
+	if err != nil {
+		pt.Close()
+		return fmt.Errorf("encryptBackup: error reading unencrypted backup file: %w", err)
+	}
+	_, err = pt.Write(unencrypted)
+	if err != nil {
+		pt.Close()
+		return fmt.Errorf("encryptBackup: error writing backup contents: %w", err)
+	}
+	pt.Close()
+
+	gpgFile := fmt.Sprintf("%s.gpg", s.file)
+	if err := ioutil.WriteFile(gpgFile, buf.Bytes(), os.ModeAppend); err != nil {
+		return fmt.Errorf("encryptBackup: error writing encrypted version of backup: %w", err)
+	}
+
+	if err := os.Remove(s.file); err != nil {
+		return fmt.Errorf("encryptBackup: error removing unencrpyted backup: %w", err)
+	}
+
+	s.file = gpgFile
+	s.logger.Infof("Encrypted backup using given passphrase, saving as `%s`.", s.file)
+	return nil
+}
+
+// copyBackup makes sure the backup file is copied to both local and remote locations
+// as per the given configuration.
+func (s *script) copyBackup() error {
+	_, name := path.Split(s.file)
+	if s.bucket != "" {
+		_, err := s.mc.FPutObject(s.ctx, s.bucket, name, s.file, minio.PutObjectOptions{
+			ContentType: "application/tar+gzip",
+		})
+		if err != nil {
+			return fmt.Errorf("copyBackup: error uploading backup to remote storage: %w", err)
+		}
+		s.logger.Infof("Uploaded a copy of backup `%s` to bucket `%s`", s.file, s.bucket)
+	}
+
+	if _, err := os.Stat(s.archive); !os.IsNotExist(err) {
+		if err := copy(s.file, path.Join(s.archive, name)); err != nil {
+			return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
+		}
+		s.logger.Infof("Stored copy of backup `%s` in local archive `%s`", s.file, s.archive)
+	}
+	return nil
+}
+
+// cleanBackup removes the backup file from disk.
+func (s *script) cleanBackup() error {
+	if err := os.Remove(s.file); err != nil {
+		return fmt.Errorf("cleanBackup: error removing file: %w", err)
+	}
+	s.logger.Info("Cleaned up local artifacts.")
+	return nil
+}
+
+// pruneOldBackups rotates away backups from local and remote storages using
+// the given configuration. In case the given configuration would delete all
+// backups, it does nothing instead.
+func (s *script) pruneOldBackups() error {
+	if s.retentionDays == nil {
+		return nil
+	}
+
+	if s.leeway != nil {
+		s.logger.Infof("Sleeping for %s before pruning backups.", s.leeway)
+		time.Sleep(*s.leeway)
+	}
+
+	s.logger.Infof("Trying to prune backups older than %d day(s) now.", *s.retentionDays)
+	deadline := s.start.AddDate(0, 0, -*s.retentionDays)
+
+	if s.bucket != "" {
+		candidates := s.mc.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{
+			WithMetadata: true,
+			Prefix:       s.pruningPrefix,
+		})
+
+		var matches []minio.ObjectInfo
+		var lenCandidates int
+		for candidate := range candidates {
+			lenCandidates++
+			if candidate.Err != nil {
+				return fmt.Errorf("pruneOldBackups: error looking up candidates from remote storage: %w", candidate.Err)
+			}
+			if candidate.LastModified.Before(deadline) {
+				matches = append(matches, candidate)
+			}
+		}
+
+		if len(matches) != 0 && len(matches) != lenCandidates {
+			objectsCh := make(chan minio.ObjectInfo)
+			go func() {
+				for _, match := range matches {
+					objectsCh <- match
+				}
+				close(objectsCh)
+			}()
+			errChan := s.mc.RemoveObjects(s.ctx, s.bucket, objectsCh, minio.RemoveObjectsOptions{})
+			var errors []error
+			for result := range errChan {
+				if result.Err != nil {
+					errors = append(errors, result.Err)
+				}
+			}
+
+			if len(errors) != 0 {
+				return fmt.Errorf(
+					"pruneOldBackups: %d error(s) removing files from remote storage: %w",
+					len(errors),
+					errors[0],
+				)
+			}
+			s.logger.Infof(
+				"Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period.",
+				len(matches),
+				lenCandidates,
+			)
+		} else if len(matches) != 0 && len(matches) == lenCandidates {
+			s.logger.Warnf(
+				"The current configuration would delete all %d remote backup copies. Refusing to do so, please check your configuration.",
+				len(matches),
+			)
+		} else {
+			s.logger.Infof("None of %d remote backup(s) were pruned.", lenCandidates)
+		}
+	}
+
+	if _, err := os.Stat(s.archive); !os.IsNotExist(err) {
+		candidates, err := filepath.Glob(
+			path.Join(s.archive, fmt.Sprintf("%s*", s.pruningPrefix)),
+		)
+		if err != nil {
+			return fmt.Errorf(
+				"pruneOldBackups: error looking up matching files, starting with: %w", err,
+			)
+		}
+
+		var matches []os.FileInfo
+		for _, candidate := range candidates {
+			fi, err := os.Stat(candidate)
+			if err != nil {
+				return fmt.Errorf(
+					"pruneOldBackups: error calling stat on file %s: %w",
+					candidate,
+					err,
+				)
+			}
+
+			if fi.ModTime().Before(deadline) {
+				matches = append(matches, fi)
+			}
+		}
+
+		if len(matches) != 0 && len(matches) != len(candidates) {
+			var errors []error
+			for _, candidate := range matches {
+				if err := os.Remove(candidate.Name()); err != nil {
+					errors = append(errors, err)
+				}
+			}
+			if len(errors) != 0 {
+				return fmt.Errorf(
+					"pruneOldBackups: %d error(s) deleting local files, starting with: %w",
+					len(errors),
+					errors[0],
+				)
+			}
+			s.logger.Infof(
+				"Pruned %d out of %d local backup(s) as their age exceeded the configured retention period.",
+				len(matches),
+				len(candidates),
+			)
+		} else if len(matches) != 0 && len(matches) == len(candidates) {
+			s.logger.Warnf(
+				"The current configuration would delete all %d local backup copies. Refusing to do so, please check your configuration.",
+				len(matches),
+			)
+		} else {
+			s.logger.Infof("None of %d local backup(s) were pruned.", len(candidates))
+		}
+	}
+	return nil
+}
+
+func (s *script) must(err error) {
+	if err != nil {
+		if s.logger == nil {
+			panic(err)
+		}
+		s.logger.Errorf("Fatal error running backup: %s", err)
+		os.Exit(1)
+	}
+}
+
+func copy(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
 }

cmd/backup/notifications.go

@@ -1,105 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-package main
-
-import (
-	"bytes"
-	_ "embed"
-	"fmt"
-	"text/template"
-	"time"
-
-	sTypes "github.com/containrrr/shoutrrr/pkg/types"
-)
-
-//go:embed notifications.tmpl
-var defaultNotifications string
-
-// NotificationData data to be passed to the notification templates
-type NotificationData struct {
-	Error  error
-	Config *Config
-	Stats  *Stats
-}
-
-// notify sends a notification using the given title and body templates.
-// Automatically creates notification data, adding the given error
-func (s *script) notify(titleTemplate string, bodyTemplate string, err error) error {
-	params := NotificationData{
-		Error:  err,
-		Stats:  s.stats,
-		Config: s.c,
-	}
-
-	titleBuf := &bytes.Buffer{}
-	if err := s.template.ExecuteTemplate(titleBuf, titleTemplate, params); err != nil {
-		return fmt.Errorf("notifyFailure: error executing %s template: %w", titleTemplate, err)
-	}
-
-	bodyBuf := &bytes.Buffer{}
-	if err := s.template.ExecuteTemplate(bodyBuf, bodyTemplate, params); err != nil {
-		return fmt.Errorf("notifyFailure: error executing %s template: %w", bodyTemplate, err)
-	}
-
-	if err := s.sendNotification(titleBuf.String(), bodyBuf.String()); err != nil {
-		return fmt.Errorf("notifyFailure: error notifying: %w", err)
-	}
-	return nil
-}
-
-// notifyFailure sends a notification about a failed backup run
-func (s *script) notifyFailure(err error) error {
-	return s.notify("title_failure", "body_failure", err)
-}
-
-// notifyFailure sends a notification about a successful backup run
-func (s *script) notifySuccess() error {
-	return s.notify("title_success", "body_success", nil)
-}
-
-// sendNotification sends a notification to all configured third party services
-func (s *script) sendNotification(title, body string) error {
-	var errs []error
-	for _, result := range s.sender.Send(body, &sTypes.Params{"title": title}) {
-		if result != nil {
-			errs = append(errs, result)
-		}
-	}
-	if len(errs) != 0 {
-		return fmt.Errorf("sendNotification: error sending message: %w", join(errs...))
-	}
-	return nil
-}
-
-var templateHelpers = template.FuncMap{
-	"formatTime": func(t time.Time) string {
-		return t.Format(time.RFC3339)
-	},
-	"formatBytesDec": func(bytes uint64) string {
-		return formatBytes(bytes, true)
-	},
-	"formatBytesBin": func(bytes uint64) string {
-		return formatBytes(bytes, false)
-	},
-}
-
-// formatBytes converts an amount of bytes in a human-readable representation
-// the decimal parameter specifies if using powers of 1000 (decimal) or powers of 1024 (binary)
-func formatBytes(b uint64, decimal bool) string {
-	unit := uint64(1024)
-	format := "%.1f %ciB"
-	if decimal {
-		unit = uint64(1000)
-		format = "%.1f %cB"
-	}
-	if b < unit {
-		return fmt.Sprintf("%d B", b)
-	}
-	div, exp := unit, 0
-	for n := b / unit; n >= unit; n /= unit {
-		div *= unit
-		exp++
-	}
-	return fmt.Sprintf(format, float64(b)/float64(div), "kMGTPE"[exp])
-}

cmd/backup/notifications.tmpl

@@ -1,26 +0,0 @@
-{{ define "title_failure" -}}
-Failure running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
-{{- end }}
-
-
-{{ define "body_failure" -}}
-Running docker-volume-backup failed with error: {{ .Error }}
-
-Log output of the failed run was:
-
-{{ .Stats.LogOutput }}
-{{- end }}
-
-
-{{ define "title_success" -}}
-Success running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
-{{- end }}
-
-
-{{ define "body_success" -}}
-Running docker-volume-backup succeeded.
-
-Log output was:
-
-{{ .Stats.LogOutput }}
-{{- end }}

cmd/backup/script.go

@@ -1,725 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-	"text/template"
-	"time"
-
-	"github.com/containrrr/shoutrrr"
-	"github.com/containrrr/shoutrrr/pkg/router"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/api/types/swarm"
-	"github.com/docker/docker/client"
-	"github.com/kelseyhightower/envconfig"
-	"github.com/leekchan/timeutil"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/otiai10/copy"
-	"github.com/sirupsen/logrus"
-	"github.com/studio-b12/gowebdav"
-	"golang.org/x/crypto/openpgp"
-)
-
-// script holds all the stateful information required to orchestrate a
-// single backup run.
-type script struct {
-	cli          *client.Client
-	minioClient  *minio.Client
-	webdavClient *gowebdav.Client
-	logger       *logrus.Logger
-	sender       *router.ServiceRouter
-	template     *template.Template
-	hooks        []hook
-	hookLevel    hookLevel
-
-	file  string
-	stats *Stats
-
-	encounteredLock bool
-
-	c *Config
-}
-
-// newScript creates all resources needed for the script to perform actions against
-// remote resources like the Docker engine or remote storage locations. All
-// reading from env vars or other configuration sources is expected to happen
-// in this method.
-func newScript() (*script, error) {
-	stdOut, logBuffer := buffer(os.Stdout)
-	s := &script{
-		c: &Config{},
-		logger: &logrus.Logger{
-			Out:       stdOut,
-			Formatter: new(logrus.TextFormatter),
-			Hooks:     make(logrus.LevelHooks),
-			Level:     logrus.InfoLevel,
-		},
-		stats: &Stats{
-			StartTime: time.Now(),
-			LogOutput: logBuffer,
-			Storages:  StoragesStats{},
-		},
-	}
-
-	s.registerHook(hookLevelPlumbing, func(error) error {
-		s.stats.EndTime = time.Now()
-		s.stats.TookTime = s.stats.EndTime.Sub(s.stats.StartTime)
-		return nil
-	})
-
-	if err := envconfig.Process("", s.c); err != nil {
-		return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
-	}
-
-	s.file = path.Join("/tmp", s.c.BackupFilename)
-	if s.c.BackupFilenameExpand {
-		s.file = os.ExpandEnv(s.file)
-		s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
-		s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix)
-	}
-	s.file = timeutil.Strftime(&s.stats.StartTime, s.file)
-
-	_, err := os.Stat("/var/run/docker.sock")
-	_, dockerHostSet := os.LookupEnv("DOCKER_HOST")
-	if !os.IsNotExist(err) || dockerHostSet {
-		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-		if err != nil {
-			return nil, fmt.Errorf("newScript: failed to create docker client")
-		}
-		s.cli = cli
-	}
-
-	if s.c.AwsS3BucketName != "" {
-		var creds *credentials.Credentials
-		if s.c.AwsAccessKeyID != "" && s.c.AwsSecretAccessKey != "" {
-			creds = credentials.NewStaticV4(
-				s.c.AwsAccessKeyID,
-				s.c.AwsSecretAccessKey,
-				"",
-			)
-		} else if s.c.AwsIamRoleEndpoint != "" {
-			creds = credentials.NewIAM(s.c.AwsIamRoleEndpoint)
-		} else {
-			return nil, errors.New("newScript: AWS_S3_BUCKET_NAME is defined, but no credentials were provided")
-		}
-
-		options := minio.Options{
-			Creds:  creds,
-			Secure: s.c.AwsEndpointProto == "https",
-		}
-
-		if s.c.AwsEndpointInsecure {
-			if !options.Secure {
-				return nil, errors.New("newScript: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
-			}
-
-			transport, err := minio.DefaultTransport(true)
-			if err != nil {
-				return nil, fmt.Errorf("newScript: failed to create default minio transport")
-			}
-			transport.TLSClientConfig.InsecureSkipVerify = true
-			options.Transport = transport
-		}
-
-		mc, err := minio.New(s.c.AwsEndpoint, &options)
-		if err != nil {
-			return nil, fmt.Errorf("newScript: error setting up minio client: %w", err)
-		}
-		s.minioClient = mc
-	}
-
-	if s.c.WebdavUrl != "" {
-		if s.c.WebdavUsername == "" || s.c.WebdavPassword == "" {
-			return nil, errors.New("newScript: WEBDAV_URL is defined, but no credentials were provided")
-		} else {
-			webdavClient := gowebdav.NewClient(s.c.WebdavUrl, s.c.WebdavUsername, s.c.WebdavPassword)
-			s.webdavClient = webdavClient
-			if s.c.WebdavUrlInsecure {
-				defaultTransport, ok := http.DefaultTransport.(*http.Transport)
-				if !ok {
-					return nil, errors.New("newScript: unexpected error when asserting type for http.DefaultTransport")
-				}
-				webdavTransport := defaultTransport.Clone()
-				webdavTransport.TLSClientConfig.InsecureSkipVerify = s.c.WebdavUrlInsecure
-				s.webdavClient.SetTransport(webdavTransport)
-			}
-		}
-	}
-
-	if s.c.EmailNotificationRecipient != "" {
-		emailURL := fmt.Sprintf(
-			"smtp://%s:%s@%s:%d/?from=%s&to=%s",
-			s.c.EmailSMTPUsername,
-			s.c.EmailSMTPPassword,
-			s.c.EmailSMTPHost,
-			s.c.EmailSMTPPort,
-			s.c.EmailNotificationSender,
-			s.c.EmailNotificationRecipient,
-		)
-		s.c.NotificationURLs = append(s.c.NotificationURLs, emailURL)
-		s.logger.Warn(
-			"Using EMAIL_* keys for providing notification configuration has been deprecated and will be removed in the next major version.",
-		)
-		s.logger.Warn(
-			"Please use NOTIFICATION_URLS instead. Refer to the README for an upgrade guide.",
-		)
-	}
-
-	hookLevel, ok := hookLevels[s.c.NotificationLevel]
-	if !ok {
-		return nil, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel)
-	}
-	s.hookLevel = hookLevel
-
-	if len(s.c.NotificationURLs) > 0 {
-		sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...)
-		if senderErr != nil {
-			return nil, fmt.Errorf("newScript: error creating sender: %w", senderErr)
-		}
-		s.sender = sender
-
-		tmpl := template.New("")
-		tmpl.Funcs(templateHelpers)
-		tmpl, err = tmpl.Parse(defaultNotifications)
-		if err != nil {
-			return nil, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
-		}
-
-		if fi, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil && fi.IsDir() {
-			tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*")
-			if err != nil {
-				return nil, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
-			}
-		}
-		s.template = tmpl
-
-		// To prevent duplicate notifications, ensure the regsistered callbacks
-		// run mutually exclusive.
-		s.registerHook(hookLevelError, func(err error) error {
-			if err == nil {
-				return nil
-			}
-			return s.notifyFailure(err)
-		})
-		s.registerHook(hookLevelInfo, func(err error) error {
-			if err != nil {
-				return nil
-			}
-			return s.notifySuccess()
-		})
-	}
-
-	return s, nil
-}
-
-func (s *script) runCommands() (func() error, error) {
-	if s.cli == nil {
-		return noop, nil
-	}
-
-	if err := s.runLabeledCommands("docker-volume-backup.exec-pre"); err != nil {
-		return noop, fmt.Errorf("runCommands: error running pre commands: %w", err)
-	}
-	return func() error {
-		if err := s.runLabeledCommands("docker-volume-backup.exec-post"); err != nil {
-			return fmt.Errorf("runCommands: error running post commands: %w", err)
-		}
-		return nil
-	}, nil
-}
-
-// stopContainers stops all Docker containers that are marked as to being
-// stopped during the backup and returns a function that can be called to
-// restart everything that has been stopped.
-func (s *script) stopContainers() (func() error, error) {
-	if s.cli == nil {
-		return noop, nil
-	}
-
-	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Quiet: true,
-	})
-	if err != nil {
-		return noop, fmt.Errorf("stopContainersAndRun: error querying for containers: %w", err)
-	}
-
-	containerLabel := fmt.Sprintf(
-		"docker-volume-backup.stop-during-backup=%s",
-		s.c.BackupStopContainerLabel,
-	)
-	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Quiet: true,
-		Filters: filters.NewArgs(filters.KeyValuePair{
-			Key:   "label",
-			Value: containerLabel,
-		}),
-	})
-
-	if err != nil {
-		return noop, fmt.Errorf("stopContainersAndRun: error querying for containers to stop: %w", err)
-	}
-
-	if len(containersToStop) == 0 {
-		return noop, nil
-	}
-
-	s.logger.Infof(
-		"Stopping %d container(s) labeled `%s` out of %d running container(s).",
-		len(containersToStop),
-		containerLabel,
-		len(allContainers),
-	)
-
-	var stoppedContainers []types.Container
-	var stopErrors []error
-	for _, container := range containersToStop {
-		if err := s.cli.ContainerStop(context.Background(), container.ID, nil); err != nil {
-			stopErrors = append(stopErrors, err)
-		} else {
-			stoppedContainers = append(stoppedContainers, container)
-		}
-	}
-
-	var stopError error
-	if len(stopErrors) != 0 {
-		stopError = fmt.Errorf(
-			"stopContainersAndRun: %d error(s) stopping containers: %w",
-			len(stopErrors),
-			join(stopErrors...),
-		)
-	}
-
-	s.stats.Containers = ContainersStats{
-		All:     uint(len(allContainers)),
-		ToStop:  uint(len(containersToStop)),
-		Stopped: uint(len(stoppedContainers)),
-	}
-
-	return func() error {
-		servicesRequiringUpdate := map[string]struct{}{}
-
-		var restartErrors []error
-		for _, container := range stoppedContainers {
-			if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok {
-				servicesRequiringUpdate[swarmServiceName] = struct{}{}
-				continue
-			}
-			if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil {
-				restartErrors = append(restartErrors, err)
-			}
-		}
-
-		if len(servicesRequiringUpdate) != 0 {
-			services, _ := s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
-			for serviceName := range servicesRequiringUpdate {
-				var serviceMatch swarm.Service
-				for _, service := range services {
-					if service.Spec.Name == serviceName {
-						serviceMatch = service
-						break
-					}
-				}
-				if serviceMatch.ID == "" {
-					return fmt.Errorf("stopContainersAndRun: couldn't find service with name %s", serviceName)
-				}
-				serviceMatch.Spec.TaskTemplate.ForceUpdate = 1
-				if _, err := s.cli.ServiceUpdate(
-					context.Background(), serviceMatch.ID,
-					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},
-				); err != nil {
-					restartErrors = append(restartErrors, err)
-				}
-			}
-		}
-
-		if len(restartErrors) != 0 {
-			return fmt.Errorf(
-				"stopContainersAndRun: %d error(s) restarting containers and services: %w",
-				len(restartErrors),
-				join(restartErrors...),
-			)
-		}
-		s.logger.Infof(
-			"Restarted %d container(s) and the matching service(s).",
-			len(stoppedContainers),
-		)
-		return nil
-	}, stopError
-}
-
-// takeBackup creates a tar archive of the configured backup location and
-// saves it to disk.
-func (s *script) takeBackup() error {
-	backupSources := s.c.BackupSources
-
-	if s.c.BackupFromSnapshot {
-		s.logger.Warn(
-			"Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.",
-		)
-		s.logger.Warn(
-			"Please use `exec-pre` and `exec-post` commands to prepare your backup sources. Refer to the README for an upgrade guide.",
-		)
-		backupSources = filepath.Join("/tmp", s.c.BackupSources)
-		// copy before compressing guard against a situation where backup folder's content are still growing.
-		s.registerHook(hookLevelPlumbing, func(error) error {
-			if err := remove(backupSources); err != nil {
-				return fmt.Errorf("takeBackup: error removing snapshot: %w", err)
-			}
-			s.logger.Infof("Removed snapshot `%s`.", backupSources)
-			return nil
-		})
-		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
-			PreserveTimes: true,
-			PreserveOwner: true,
-		}); err != nil {
-			return fmt.Errorf("takeBackup: error creating snapshot: %w", err)
-		}
-		s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources)
-	}
-
-	tarFile := s.file
-	s.registerHook(hookLevelPlumbing, func(error) error {
-		if err := remove(tarFile); err != nil {
-			return fmt.Errorf("takeBackup: error removing tar file: %w", err)
-		}
-		s.logger.Infof("Removed tar file `%s`.", tarFile)
-		return nil
-	})
-
-	backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources))
-	if err != nil {
-		return fmt.Errorf("takeBackup: error getting absolute path: %w", err)
-	}
-
-	var filesEligibleForBackup []string
-	if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) {
-			return nil
-		}
-		filesEligibleForBackup = append(filesEligibleForBackup, path)
-		return nil
-	}); err != nil {
-		return fmt.Errorf("compress: error walking filesystem tree: %w", err)
-	}
-
-	if err := createArchive(filesEligibleForBackup, backupSources, tarFile); err != nil {
-		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
-	}
-
-	s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, tarFile)
-	return nil
-}
-
-// encryptBackup encrypts the backup file using PGP and the configured passphrase.
-// In case no passphrase is given it returns early, leaving the backup file
-// untouched.
-func (s *script) encryptBackup() error {
-	if s.c.GpgPassphrase == "" {
-		return nil
-	}
-
-	gpgFile := fmt.Sprintf("%s.gpg", s.file)
-	s.registerHook(hookLevelPlumbing, func(error) error {
-		if err := remove(gpgFile); err != nil {
-			return fmt.Errorf("encryptBackup: error removing gpg file: %w", err)
-		}
-		s.logger.Infof("Removed GPG file `%s`.", gpgFile)
-		return nil
-	})
-
-	outFile, err := os.Create(gpgFile)
-	defer outFile.Close()
-	if err != nil {
-		return fmt.Errorf("encryptBackup: error opening out file: %w", err)
-	}
-
-	_, name := path.Split(s.file)
-	dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
-		IsBinary: true,
-		FileName: name,
-	}, nil)
-	defer dst.Close()
-	if err != nil {
-		return fmt.Errorf("encryptBackup: error encrypting backup file: %w", err)
-	}
-
-	src, err := os.Open(s.file)
-	if err != nil {
-		return fmt.Errorf("encryptBackup: error opening backup file `%s`: %w", s.file, err)
-	}
-
-	if _, err := io.Copy(dst, src); err != nil {
-		return fmt.Errorf("encryptBackup: error writing ciphertext to file: %w", err)
-	}
-
-	s.file = gpgFile
-	s.logger.Infof("Encrypted backup using given passphrase, saving as `%s`.", s.file)
-	return nil
-}
-
-// copyBackup makes sure the backup file is copied to both local and remote locations
-// as per the given configuration.
-func (s *script) copyBackup() error {
-	_, name := path.Split(s.file)
-	if stat, err := os.Stat(s.file); err != nil {
-		return fmt.Errorf("copyBackup: unable to stat backup file: %w", err)
-	} else {
-		size := stat.Size()
-		s.stats.BackupFile = BackupFileStats{
-			Size:     uint64(size),
-			Name:     name,
-			FullPath: s.file,
-		}
-	}
-
-	if s.minioClient != nil {
-		if _, err := s.minioClient.FPutObject(context.Background(), s.c.AwsS3BucketName, filepath.Join(s.c.AwsS3Path, name), s.file, minio.PutObjectOptions{
-			ContentType: "application/tar+gzip",
-		}); err != nil {
-			return fmt.Errorf("copyBackup: error uploading backup to remote storage: %w", err)
-		}
-		s.logger.Infof("Uploaded a copy of backup `%s` to bucket `%s`.", s.file, s.c.AwsS3BucketName)
-	}
-
-	if s.webdavClient != nil {
-		bytes, err := os.ReadFile(s.file)
-		if err != nil {
-			return fmt.Errorf("copyBackup: error reading the file to be uploaded: %w", err)
-		}
-		if err := s.webdavClient.MkdirAll(s.c.WebdavPath, 0644); err != nil {
-			return fmt.Errorf("copyBackup: error creating directory '%s' on WebDAV server: %w", s.c.WebdavPath, err)
-		}
-		if err := s.webdavClient.Write(filepath.Join(s.c.WebdavPath, name), bytes, 0644); err != nil {
-			return fmt.Errorf("copyBackup: error uploading the file to WebDAV server: %w", err)
-		}
-		s.logger.Infof("Uploaded a copy of backup `%s` to WebDAV-URL '%s' at path '%s'.", s.file, s.c.WebdavUrl, s.c.WebdavPath)
-	}
-
-	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
-		if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
-			return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
-		}
-		s.logger.Infof("Stored copy of backup `%s` in local archive `%s`.", s.file, s.c.BackupArchive)
-		if s.c.BackupLatestSymlink != "" {
-			symlink := path.Join(s.c.BackupArchive, s.c.BackupLatestSymlink)
-			if _, err := os.Lstat(symlink); err == nil {
-				os.Remove(symlink)
-			}
-			if err := os.Symlink(name, symlink); err != nil {
-				return fmt.Errorf("copyBackup: error creating latest symlink: %w", err)
-			}
-			s.logger.Infof("Created/Updated symlink `%s` for latest backup.", s.c.BackupLatestSymlink)
-		}
-	}
-	return nil
-}
-
-// pruneBackups rotates away backups from local and remote storages using
-// the given configuration. In case the given configuration would delete all
-// backups, it does nothing instead and logs a warning.
-func (s *script) pruneBackups() error {
-	if s.c.BackupRetentionDays < 0 {
-		return nil
-	}
-
-	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
-
-	// doPrune holds general control flow that applies to any kind of storage.
-	// Callers can pass in a thunk that performs the actual deletion of files.
-	var doPrune = func(lenMatches, lenCandidates int, description string, doRemoveFiles func() error) error {
-		if lenMatches != 0 && lenMatches != lenCandidates {
-			if err := doRemoveFiles(); err != nil {
-				return err
-			}
-			s.logger.Infof(
-				"Pruned %d out of %d %s as their age exceeded the configured retention period of %d days.",
-				lenMatches,
-				lenCandidates,
-				description,
-				s.c.BackupRetentionDays,
-			)
-		} else if lenMatches != 0 && lenMatches == lenCandidates {
-			s.logger.Warnf("The current configuration would delete all %d existing %s.", lenMatches, description)
-			s.logger.Warn("Refusing to do so, please check your configuration.")
-		} else {
-			s.logger.Infof("None of %d existing %s were pruned.", lenCandidates, description)
-		}
-		return nil
-	}
-
-	if s.minioClient != nil {
-		candidates := s.minioClient.ListObjects(context.Background(), s.c.AwsS3BucketName, minio.ListObjectsOptions{
-			WithMetadata: true,
-			Prefix:       filepath.Join(s.c.AwsS3Path, s.c.BackupPruningPrefix),
-			Recursive:    true,
-		})
-
-		var matches []minio.ObjectInfo
-		var lenCandidates int
-		for candidate := range candidates {
-			lenCandidates++
-			if candidate.Err != nil {
-				return fmt.Errorf(
-					"pruneBackups: error looking up candidates from remote storage: %w",
-					candidate.Err,
-				)
-			}
-			if candidate.LastModified.Before(deadline) {
-				matches = append(matches, candidate)
-			}
-		}
-
-		s.stats.Storages.S3 = StorageStats{
-			Total:  uint(lenCandidates),
-			Pruned: uint(len(matches)),
-		}
-
-		doPrune(len(matches), lenCandidates, "remote backup(s)", func() error {
-			objectsCh := make(chan minio.ObjectInfo)
-			go func() {
-				for _, match := range matches {
-					objectsCh <- match
-				}
-				close(objectsCh)
-			}()
-			errChan := s.minioClient.RemoveObjects(context.Background(), s.c.AwsS3BucketName, objectsCh, minio.RemoveObjectsOptions{})
-			var removeErrors []error
-			for result := range errChan {
-				if result.Err != nil {
-					removeErrors = append(removeErrors, result.Err)
-				}
-			}
-			if len(removeErrors) != 0 {
-				return join(removeErrors...)
-			}
-			return nil
-		})
-	}
-
-	if s.webdavClient != nil {
-		candidates, err := s.webdavClient.ReadDir(s.c.WebdavPath)
-		if err != nil {
-			return fmt.Errorf("pruneBackups: error looking up candidates from remote storage: %w", err)
-		}
-		var matches []fs.FileInfo
-		var lenCandidates int
-		for _, candidate := range candidates {
-			if !strings.HasPrefix(candidate.Name(), s.c.BackupPruningPrefix) {
-				continue
-			}
-			lenCandidates++
-			if candidate.ModTime().Before(deadline) {
-				matches = append(matches, candidate)
-			}
-		}
-
-		s.stats.Storages.WebDAV = StorageStats{
-			Total:  uint(lenCandidates),
-			Pruned: uint(len(matches)),
-		}
-
-		doPrune(len(matches), lenCandidates, "WebDAV backup(s)", func() error {
-			for _, match := range matches {
-				if err := s.webdavClient.Remove(filepath.Join(s.c.WebdavPath, match.Name())); err != nil {
-					return fmt.Errorf("pruneBackups: error removing file from WebDAV storage: %w", err)
-				}
-			}
-			return nil
-		})
-	}
-
-	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
-		globPattern := path.Join(
-			s.c.BackupArchive,
-			fmt.Sprintf("%s*", s.c.BackupPruningPrefix),
-		)
-		globMatches, err := filepath.Glob(globPattern)
-		if err != nil {
-			return fmt.Errorf(
-				"pruneBackups: error looking up matching files using pattern %s: %w",
-				globPattern,
-				err,
-			)
-		}
-
-		var candidates []string
-		for _, candidate := range globMatches {
-			fi, err := os.Lstat(candidate)
-			if err != nil {
-				return fmt.Errorf(
-					"pruneBackups: error calling Lstat on file %s: %w",
-					candidate,
-					err,
-				)
-			}
-
-			if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
-				candidates = append(candidates, candidate)
-			}
-		}
-
-		var matches []string
-		for _, candidate := range candidates {
-			fi, err := os.Stat(candidate)
-			if err != nil {
-				return fmt.Errorf(
-					"pruneBackups: error calling stat on file %s: %w",
-					candidate,
-					err,
-				)
-			}
-			if fi.ModTime().Before(deadline) {
-				matches = append(matches, candidate)
-			}
-		}
-
-		s.stats.Storages.Local = StorageStats{
-			Total:  uint(len(candidates)),
-			Pruned: uint(len(matches)),
-		}
-
-		doPrune(len(matches), len(candidates), "local backup(s)", func() error {
-			var removeErrors []error
-			for _, match := range matches {
-				if err := os.Remove(match); err != nil {
-					removeErrors = append(removeErrors, err)
-				}
-			}
-			if len(removeErrors) != 0 {
-				return fmt.Errorf(
-					"pruneBackups: %d error(s) deleting local files, starting with: %w",
-					len(removeErrors),
-					join(removeErrors...),
-				)
-			}
-			return nil
-		})
-	}
-	return nil
-}
-
-// must exits the script run prematurely in case the given error
-// is non-nil.
-func (s *script) must(err error) {
-	if err != nil {
-		s.logger.Errorf("Fatal error running backup: %s", err)
-		panic(err)
-	}
-}

cmd/backup/stats.go

@@ -1,50 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-package main
-
-import (
-	"bytes"
-	"time"
-)
-
-// ContainersStats stats about the docker containers
-type ContainersStats struct {
-	All        uint
-	ToStop     uint
-	Stopped    uint
-	StopErrors uint
-}
-
-// BackupFileStats stats about the created backup file
-type BackupFileStats struct {
-	Name     string
-	FullPath string
-	Size     uint64
-}
-
-// StorageStats stats about the status of an archival directory
-type StorageStats struct {
-	Total       uint
-	Pruned      uint
-	PruneErrors uint
-}
-
-// StoragesStats stats about each possible archival location (Local, WebDAV, S3)
-type StoragesStats struct {
-	Local  StorageStats
-	WebDAV StorageStats
-	S3     StorageStats
-}
-
-// Stats global stats regarding script execution
-type Stats struct {
-	StartTime  time.Time
-	EndTime    time.Time
-	TookTime   time.Duration
-	LockedTime time.Duration
-	LogOutput  *bytes.Buffer
-	Containers ContainersStats
-	BackupFile BackupFileStats
-	Storages   StoragesStats
-}

cmd/backup/util.go

@@ -1,90 +0,0 @@
-// Copyright 2022 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-package main
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"strings"
-)
-
-var noop = func() error { return nil }
-
-// copy creates a copy of the file located at `dst` at `src`.
-func copyFile(src, dst string) error {
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer in.Close()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-
-	_, err = io.Copy(out, in)
-	if err != nil {
-		out.Close()
-		return err
-	}
-	return out.Close()
-}
-
-// join takes a list of errors and joins them into a single error
-func join(errs ...error) error {
-	if len(errs) == 1 {
-		return errs[0]
-	}
-	var msgs []string
-	for _, err := range errs {
-		if err == nil {
-			continue
-		}
-		msgs = append(msgs, err.Error())
-	}
-	return errors.New("[" + strings.Join(msgs, ", ") + "]")
-}
-
-// remove removes the given file or directory from disk.
-func remove(location string) error {
-	fi, err := os.Lstat(location)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return fmt.Errorf("remove: error checking for existence of `%s`: %w", location, err)
-	}
-	if fi.IsDir() {
-		err = os.RemoveAll(location)
-	} else {
-		err = os.Remove(location)
-	}
-	if err != nil {
-		return fmt.Errorf("remove: error removing `%s`: %w", location, err)
-	}
-	return nil
-}
-
-// buffer takes an io.Writer and returns a wrapped version of the
-// writer that writes to both the original target as well as the returned buffer
-func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {
-	buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w}
-	return buffering, &buffering.buf
-}
-
-type bufferingWriter struct {
-	buf    bytes.Buffer
-	writer io.Writer
-}
-
-func (b *bufferingWriter) Write(p []byte) (n int, err error) {
-	if n, err := b.buf.Write(p); err != nil {
-		return n, fmt.Errorf("bufferingWriter: error writing to buffer: %w", err)
-	}
-	return b.writer.Write(p)
-}

docs/NOTIFICATION-TEMPLATES.md

@@ -1,39 +0,0 @@
-# Notification templates reference
-
-In order to customize title and body of notifications you'll have to write a [go template](https://pkg.go.dev/text/template) and mount it inside the `/etc/dockervolumebackup/notifications.d/` directory.
-
-Configuration, data about the backup run and helper functions will be passed to this template, this page documents them fully.
-
-## Data
-Here is a list of all data passed to the template:
-
-* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`)
-* `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates
-* `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available.
-  * `StartTime`: time when the script started execution
-  * `EndTime`: time when the backup has completed successfully (after pruning)
-  * `TookTime`: amount of time it took for the backup to run. (equal to `EndTime - StartTime`)
-  * `LockedTime`: amount of time it took for the backup to acquire the exclusive lock
-  * `LogOutput`: full log of the application
-  * `Containers`: object containing stats about the docker containers
-    * `All`: total number of containers
-    * `ToStop`: number of containers matched by the stop rule
-    * `Stopped`: number of containers successfully stopped
-    * `StopErrors`: number of containers that were unable to be stopped (equal to `ToStop - Stopped`)
-  * `BackupFile`: object containing information about the backup file
-    * `Name`: name of the backup file (e.g. `backup-2022-02-11T01-00-00.tar.gz`)
-    * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
-    * `Size`: size in bytes of the backup file
-  * `Storages`: object that holds stats about each storage
-    * `Local`, `S3` or `WebDAV`:
-      * `Total`: total number of backup files
-      * `Pruned`: number of backup files that were deleted due to pruning rule
-      * `PruneErrors`: number of backup files that were unable to be pruned
-
-## Functions
-
-Some formatting functions are also available:
-
-* `formatTime`: formats a time object using [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format (e.g. `2022-02-11T01:00:00Z`)
-* `formatBytesBin`: formats an amount of bytes using powers of 1024 (e.g. `7055258` bytes will be `6.7 MiB`) 
-* `formatBytesDec`: formats an amount of bytes using powers of 1000 (e.g. `7055258` bytes will be `7.1 MB`)

entrypoint.sh

@@ -3,23 +3,38 @@
 # Copyright 2021 - Offen Authors <hioffen@posteo.de>
 # SPDX-License-Identifier: MPL-2.0
 
+# Portions of this file are taken from github.com/futurice/docker-volume-backup
+# See NOTICE for information about authors and licensing.
+
 set -e
 
-if [ ! -d "/etc/dockervolumebackup/conf.d" ]; then
+# Write cronjob env to file, fill in sensible defaults, and read them back in
-  BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+mkdir -p /etc/backup
+cat <<EOF > /etc/backup.env
+BACKUP_SOURCES="${BACKUP_SOURCES:-/backup}"
+BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+BACKUP_FILENAME="${BACKUP_FILENAME:-backup-%Y-%m-%dT%H-%M-%S.tar.gz}"
+BACKUP_ARCHIVE="${BACKUP_ARCHIVE:-/archive}"
 
-  echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
+BACKUP_RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-}"
-  echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
+BACKUP_PRUNING_LEEWAY="${BACKUP_PRUNING_LEEWAY:-1m}"
-else
+BACKUP_PRUNING_PREFIX="${BACKUP_PRUNING_PREFIX:-}"
-  echo "/etc/dockervolumebackup/conf.d was found, using configuration files from this directory."
+BACKUP_STOP_CONTAINER_LABEL="${BACKUP_STOP_CONTAINER_LABEL:-true}"
 
-  for file in /etc/dockervolumebackup/conf.d/*; do
+AWS_S3_BUCKET_NAME="${AWS_S3_BUCKET_NAME:-}"
-    source $file
+AWS_ENDPOINT="${AWS_ENDPOINT:-s3.amazonaws.com}"
-    BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+AWS_ENDPOINT_PROTO="${AWS_ENDPOINT_PROTO:-https}"
-    echo "Appending cron.d entry with expression $BACKUP_CRON_EXPRESSION and configuration file $file"
+AWS_ENDPOINT_INSECURE="${AWS_ENDPOINT_INSECURE:-}"
-    (crontab -l; echo "$BACKUP_CRON_EXPRESSION /bin/sh -c 'set -a; source $file; set +a && backup' 2>&1") | crontab -
-  done
-fi
 
+GPG_PASSPHRASE="${GPG_PASSPHRASE:-}"
+EOF
+chmod a+x /etc/backup.env
+source /etc/backup.env
+
+# Add our cron entry, and direct stdout & stderr to Docker commands stdout
+echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
+echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
+
+# Let cron take the wheel
 echo "Starting cron in foreground."
 crond -f -l 8

go.mod

@@ -1,65 +1,44 @@
 module github.com/offen/docker-volume-backup
 
-go 1.18
+go 1.17
 
 require (
-	github.com/containrrr/shoutrrr v0.5.2
+	github.com/docker/docker v20.10.8+incompatible
-	github.com/cosiner/argv v0.1.0
+	github.com/joho/godotenv v1.3.0
-	github.com/docker/docker v20.10.11+incompatible
-	github.com/gofrs/flock v0.8.1
-	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
-	github.com/minio/minio-go/v7 v7.0.16
+	github.com/minio/minio-go/v7 v7.0.12
-	github.com/otiai10/copy v1.7.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
+	github.com/walle/targz v0.0.0-20140417120357-57fe4206da5a
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
-	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
 )
 
 require (
-	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/Microsoft/go-winio v0.4.17 // indirect
-	github.com/containerd/containerd v1.6.6 // indirect
+	github.com/containerd/containerd v1.5.5 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/fatih/color v1.10.0 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.2.0 // indirect
-	github.com/gorilla/mux v1.7.3 // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid v1.3.1 // indirect
-	github.com/klauspost/compress v1.15.6 // indirect
+	github.com/minio/md5-simd v1.1.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/minio/sha256-simd v0.1.1 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/moby/term v0.0.0-20200312100748-672ec06f55cd // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	github.com/nxadm/tail v1.4.6 // indirect
-	github.com/onsi/ginkgo v1.14.2 // indirect
-	github.com/onsi/gomega v1.10.3 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rs/xid v1.3.0 // indirect
+	github.com/rs/xid v1.2.1 // indirect
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/text v0.3.4 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
-	google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect
+	google.golang.org/grpc v1.33.2 // indirect
-	google.golang.org/grpc v1.47.0 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/ini.v1 v1.57.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
-	gopkg.in/ini.v1 v1.65.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

test/cli/run.sh

@@ -7,9 +7,6 @@ cd $(dirname $0)
 docker network create test_network
 docker volume create backup_data
 docker volume create app_data
-# This volume is created to test whether empty directories are handled
-# correctly. It is not supposed to hold any data.
-docker volume create empty_data
 
 docker run -d \
   --name minio \
@@ -26,15 +23,16 @@ docker exec minio mkdir -p /data/backup
 docker run -d \
   --name offen \
   --network test_network \
+  --label "docker-volume-backup.stop-during-backup=true" \
   -v app_data:/var/opt/offen/ \
   offen/offen:latest
 
 sleep 10
 
-docker run --rm \
+docker run -d \
+  --name backup \
   --network test_network \
   -v app_data:/backup/app_data \
-  -v empty_data:/backup/empty_data \
   -v /var/run/docker.sock:/var/run/docker.sock \
   --env AWS_ACCESS_KEY_ID=test \
   --env AWS_SECRET_ACCESS_KEY=GMusLtUmILge2by+z890kQ \
@@ -42,19 +40,18 @@ docker run --rm \
   --env AWS_ENDPOINT_PROTO=http \
   --env AWS_S3_BUCKET_NAME=backup \
   --env BACKUP_FILENAME=test.tar.gz \
-  --env "BACKUP_FROM_SNAPSHOT=true" \
+  --env BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?" \
-  --entrypoint backup \
+  offen/docker-volume-backup:$TEST_VERSION
-  offen/docker-volume-backup:${TEST_VERSION:-canary}
+
+docker exec backup backup
 
 docker run --rm -it \
   -v backup_data:/data alpine \
-  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db && test -d /backup/empty_data'
+  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
 
-echo "[TEST:PASS] Found relevant files in untared remote backup."
+echo "[TEST:PASS] Found relevant files in untared backup."
 
-# This test does not stop containers during backup. This is happening on
+if [ "$(docker ps -q | wc -l)" != "3" ]; then
-# purpose in order to cover this setup as well.
-if [ "$(docker ps -q | wc -l)" != "2" ]; then
   echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
   docker ps
   exit 1
@@ -62,6 +59,6 @@ fi
 
 echo "[TEST:PASS] All containers running post backup."
 
-docker rm $(docker stop minio offen)
+docker rm $(docker stop minio offen backup)
 docker volume rm backup_data app_data
 docker network rm test_network

test/commands/.gitignore

@@ -1 +0,0 @@
-local

test/commands/docker-compose.yml

@@ -1,36 +0,0 @@
-version: '3.8'
-
-services:
-  database:
-    image: mariadb:10.7
-    deploy:
-      restart_policy:
-        condition: on-failure
-    environment:
-      MARIADB_ROOT_PASSWORD: test
-      MARIADB_DATABASE: backup
-    labels:
-      - docker-volume-backup.exec-pre=/bin/sh -c 'mysqldump -ptest --all-databases > /tmp/volume/dump.sql'
-      - docker-volume-backup.exec-post=/bin/sh -c 'echo "post" > /tmp/volume/post.txt'
-      - docker-volume-backup.exec-label=test
-    volumes:
-      - app_data:/tmp/volume
-
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    deploy:
-      restart_policy:
-        condition: on-failure
-    environment:
-      BACKUP_FILENAME: test.tar.gz
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      EXEC_LABEL: test
-      EXEC_FORWARD_OUTPUT: "true"
-    volumes:
-      - archive:/archive
-      - app_data:/backup/data:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-
-volumes:
-  app_data:
-  archive:

test/commands/run.sh

@@ -1,62 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(dirname $0)
-
-
-docker-compose up -d
-sleep 30 # mariadb likes to take a bit before responding
-
-docker-compose exec backup backup
-sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' commands_archive) ./local
-
-tar -xvf ./local/test.tar.gz
-if [ ! -f ./backup/data/dump.sql ]; then
-  echo "[TEST:FAIL] Could not find file written by pre command."
-  exit 1
-fi
-echo "[TEST:PASS] Found expected file."
-
-if [ -f ./backup/data/post.txt ]; then
-  echo "[TEST:FAIL] File created in post command was present in backup."
-  exit 1
-fi
-echo "[TEST:PASS] Did not find unexpected file."
-
-docker-compose down --volumes
-sudo rm -rf ./local
-
-
-echo "[TEST:INFO] Running commands test in swarm mode next."
-
-docker swarm init
-
-docker stack deploy --compose-file=docker-compose.yml test_stack
-
-while [ -z $(docker ps -q -f name=backup) ]; do
-  echo "[TEST:INFO] Backup container not ready yet. Retrying."
-  sleep 1
-done
-
-sleep 20
-
-docker exec $(docker ps -q -f name=backup) backup
-
-sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' test_stack_archive) ./local
-
-tar -xvf ./local/test.tar.gz
-if [ ! -f ./backup/data/dump.sql ]; then
-  echo "[TEST:FAIL] Could not find file written by pre command."
-  exit 1
-fi
-echo "[TEST:PASS] Found expected file."
-
-if [ -f ./backup/data/post.txt ]; then
-  echo "[TEST:FAIL] File created in post command was present in backup."
-  exit 1
-fi
-echo "[TEST:PASS] Did not find unexpected file."
-
-docker stack rm test_stack
-docker swarm leave --force

test/compose/docker-compose.yml

@@ -10,23 +10,12 @@ services:
       MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
     entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
     volumes:
-      - minio_backup_data:/data
+      - backup_data:/data
 
-  webdav:
+  backup: &default_backup_service
-    image: bytemark/webdav:2.4
+    image: offen/docker-volume-backup:${TEST_VERSION}
-    environment:
-      AUTH_TYPE: Digest
-      USERNAME: test
-      PASSWORD: test
-    volumes:
-      - webdav_backup_data:/var/lib/dav
-
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    hostname: hostnametoken
     depends_on:
       - minio
-      - webdav
     restart: always
     environment:
       AWS_ACCESS_KEY_ID: test
@@ -34,19 +23,11 @@ services:
       AWS_ENDPOINT: minio:9000
       AWS_ENDPOINT_PROTO: http
       AWS_S3_BUCKET_NAME: backup
-      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test.tar.gz
-      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
-      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
-      GPG_PASSPHRASE: 1234secret
-      WEBDAV_URL: http://webdav/
-      WEBDAV_URL_INSECURE: 'true'
-      WEBDAV_PATH: /my/new/path/
-      WEBDAV_USERNAME: test
-      WEBDAV_PASSWORD: test
     volumes:
       - ./local:/archive
       - app_data:/backup/app_data:ro
@@ -59,7 +40,7 @@ services:
     volumes:
       - app_data:/var/opt/offen
 
+
 volumes:
-  minio_backup_data:
+  backup_data:
-  webdav_backup_data:
   app_data:

test/compose/run.sh

@@ -9,38 +9,27 @@ mkdir -p local
 docker-compose up -d
 sleep 5
 
-# A symlink for a known file in the volume is created so the test can check
-# whether symlinks are preserved on backup.
-docker-compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link
 docker-compose exec backup backup
 
-sleep 5
+docker run --rm -it \
-if [ "$(docker-compose ps -q | wc -l)" != "4" ]; then
+  -v compose_backup_data:/data alpine \
+  ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
+
+echo "[TEST:PASS] Found relevant files in untared remote backup."
+
+tar -xf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
+
+echo "[TEST:PASS] Found relevant files in untared local backup."
+
+if [ "$(docker-compose ps -q | wc -l)" != "3" ]; then
   echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
   docker-compose ps
   exit 1
 fi
+
 echo "[TEST:PASS] All containers running post backup."
 
-
+docker-compose down
-docker run --rm -it \
-  -v compose_minio_backup_data:/minio_data \
-  -v compose_webdav_backup_data:/webdav_data alpine \
-  ash -c 'apk add gnupg && \
-          echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /minio_data/backup/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xvf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db && \
-          echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /webdav_data/data/my/new/path/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xvf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
-
-echo "[TEST:PASS] Found relevant files in decrypted and untared remote backups."
-
-echo 1234secret | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 ./local/test-hostnametoken.tar.gz.gpg > ./local/decrypted.tar.gz
-tar -xf ./local/decrypted.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
-rm ./local/decrypted.tar.gz
-test -L /tmp/backup/app_data/db.link
-
-echo "[TEST:PASS] Found relevant files in decrypted and untared local backup."
-
-test -L ./local/test-hostnametoken.latest.tar.gz.gpg
-echo "[TEST:PASS] Found symlink to latest version in local backup."
 
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
@@ -51,18 +40,16 @@ sleep 5
 docker-compose exec backup backup
 
 docker run --rm -it \
-  -v compose_minio_backup_data:/minio_data \
+  -v compose_backup_data:/data alpine \
-  -v compose_webdav_backup_data:/webdav_data alpine \
+  ash -c '[ $(find /data/backup/ -type f | wc -l) = "1" ]'
-  ash -c '[ $(find /minio_data/backup/ -type f | wc -l) = "1" ] && \
-          [ $(find /webdav_data/data/my/new/path/ -type f | wc -l) = "1" ]'
 
 echo "[TEST:PASS] Remote backups have not been deleted."
 
 if [ "$(find ./local -type f | wc -l)" != "1" ]; then
   echo "[TEST:FAIL] Backups should not have been deleted, instead seen:"
   find ./local -type f
-  exit 1
 fi
+
 echo "[TEST:PASS] Local backups have not been deleted."
 
 docker-compose down --volumes

test/confd/.gitignore

@@ -1 +0,0 @@
-local

test/confd/01backup.env

@@ -1,2 +0,0 @@
-BACKUP_FILENAME="conf.tar.gz"
-BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/02backup.env

@@ -1,2 +0,0 @@
-BACKUP_FILENAME="other.tar.gz"
-BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/03never.env

@@ -1,2 +0,0 @@
-BACKUP_FILENAME="never.tar.gz"
-BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?"

test/confd/docker-compose.yml

@@ -1,23 +0,0 @@
-version: '3'
-
-services:
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    restart: always
-    volumes:
-      - ./local:/archive
-      - app_data:/backup/app_data:ro
-      - ./01backup.env:/etc/dockervolumebackup/conf.d/01backup.env
-      - ./02backup.env:/etc/dockervolumebackup/conf.d/02backup.env
-      - ./03never.env:/etc/dockervolumebackup/conf.d/03never.env
-      - /var/run/docker.sock:/var/run/docker.sock
-
-  offen:
-    image: offen/offen:latest
-    labels:
-      - docker-volume-backup.stop-during-backup=true
-    volumes:
-      - app_data:/var/opt/offen
-
-volumes:
-  app_data:

test/confd/run.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(dirname $0)
-
-mkdir -p local
-
-docker-compose up -d
-
-# sleep until a backup is guaranteed to have happened on the 1 minute schedule
-sleep 100
-
-docker-compose down --volumes
-
-if [ ! -f ./local/conf.tar.gz ]; then
-  echo "[TEST:FAIL] Config from file was not used."
-  exit 1
-fi
-echo "[TEST:PASS] Config from file was used."
-
-if [ ! -f ./local/other.tar.gz ]; then
-  echo "[TEST:FAIL] Run on same schedule did not succeed."
-  exit 1
-fi
-echo "[TEST:PASS] Run on same schedule succeeded."
-
-if [ -f ./local/never.tar.gz ]; then
-  echo "[TEST:FAIL] Unexpected file was found."
-  exit 1
-fi
-echo "[TEST:PASS] Unexpected cron did not run."

test/ignore/.gitignore

@@ -1 +0,0 @@
-local

test/ignore/docker-compose.yml

@@ -1,15 +0,0 @@
-version: '3.8'
-
-services:
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    deploy:
-      restart_policy:
-        condition: on-failure
-    environment:
-      BACKUP_FILENAME: test.tar.gz
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_EXCLUDE_REGEXP: '\.(me|you)$$'
-    volumes:
-      - ./local:/archive
-      - ./sources:/backup/data:ro

test/ignore/run.sh

@@ -1,27 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(dirname $0)
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-docker-compose exec backup backup
-
-docker-compose down --volumes
-
-out=$(mktemp -d)
-sudo tar --same-owner -xvf ./local/test.tar.gz -C "$out"
-
-if [ ! -f "$out/backup/data/me.txt" ]; then
-  echo "[TEST:FAIL] Expected file was not found."
-  exit 1
-fi
-echo "[TEST:PASS] Expected file was found."
-
-if [ -f "$out/backup/data/skip.me" ]; then
-  echo "[TEST:FAIL] Ignored file was found."
-  exit 1
-fi
-echo "[TEST:PASS] Ignored file was not found."

test/notifications/.gitignore

@@ -1 +0,0 @@
-local

test/notifications/docker-compose.yml

@@ -1,36 +0,0 @@
-version: '3'
-
-services:
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    restart: always
-    environment:
-      BACKUP_FILENAME: test.tar.gz
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_PRUNING_PREFIX: test
-      NOTIFICATION_LEVEL: info
-      NOTIFICATION_URLS: ${NOTIFICATION_URLS}
-    volumes:
-      - ./local:/archive
-      - app_data:/backup/app_data:ro
-      - ./notifications.tmpl:/etc/dockervolumebackup/notifications.d/notifications.tmpl
-
-  offen:
-    image: offen/offen:latest
-    labels:
-      - docker-volume-backup.stop-during-backup=true
-    volumes:
-      - app_data:/var/opt/offen
-
-  gotify:
-    image: gotify/server
-    ports:
-      - 8080:80
-    environment:
-      - GOTIFY_DEFAULTUSER_PASS=custom
-    volumes:
-      - gotify_data:/app/data
-
-volumes:
-  app_data:
-  gotify_data:

test/notifications/notifications.tmpl

@@ -1,7 +0,0 @@
-{{ define "title_success" -}}
-Successful test run, yay!
-{{- end }}
-
-{{ define "body_success" -}}
-Backing up {{ .Stats.BackupFile.FullPath }} succeeded.
-{{- end }}

test/notifications/run.sh

@@ -1,52 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(dirname $0)
-
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-
-GOTIFY_TOKEN=$(curl -sSLX POST -H 'Content-Type: application/json' -d '{"name":"test"}' http://admin:custom@localhost:8080/application | jq -r '.token')
-echo "[TEST:INFO] Set up Gotify application using token $GOTIFY_TOKEN"
-
-docker-compose exec backup backup
-
-NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
-if [ "$NUM_MESSAGES" != 0 ]; then
-  echo "[TEST:FAIL] Expected no notifications to be sent when not configured"
-  exit 1
-fi
-echo "[TEST:PASS] No notifications were sent when not configured."
-
-docker-compose down
-
-NOTIFICATION_URLS="gotify://gotify/${GOTIFY_TOKEN}?disableTLS=true" docker-compose up -d
-
-docker-compose exec backup backup
-
-NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
-if [ "$NUM_MESSAGES" != 1 ]; then
-  echo "[TEST:FAIL] Expected one notifications to be sent when configured"
-  exit 1
-fi
-echo "[TEST:PASS] Correct number of notifications were sent when configured."
-
-MESSAGE_TITLE=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].title')
-MESSAGE_BODY=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].message')
-
-if [ "$MESSAGE_TITLE" != "Successful test run, yay!" ]; then
-  echo "[TEST:FAIL] Unexpected notification title $MESSAGE_TITLE"
-  exit 1
-fi
-echo "[TEST:PASS] Custom notification title was used."
-
-if [ "$MESSAGE_BODY" != "Backing up /tmp/test.tar.gz succeeded." ]; then
-  echo "[TEST:FAIL] Unexpected notification body $MESSAGE_BODY"
-  exit 1
-fi
-echo "[TEST:PASS] Custom notification body was used."
-
-docker-compose down --volumes

test/ownership/.gitignore

@@ -1 +0,0 @@
-local

test/ownership/docker-compose.yml

@@ -1,27 +0,0 @@
-version: '3'
-
-services:
-  db:
-    image: postgres:14-alpine
-    restart: unless-stopped
-    labels:
-      - docker-volume-backup.stop-during-backup=true
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_PASSWORD=1FHJMSwt0yhIN1zS7I4DilGUhThBKq0x
-      - POSTGRES_USER=test
-      - POSTGRES_DB=test
-
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION}
-    restart: always
-    environment:
-      BACKUP_FILENAME: backup.tar.gz
-    volumes:
-      - postgres_data:/backup/postgres:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - ./local:/archive
-
-volumes:
-  postgres_data:

test/ownership/run.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-# This test refers to https://github.com/offen/docker-volume-backup/issues/71
-
-set -e
-
-cd $(dirname $0)
-
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-sudo tar --same-owner -xvf ./local/backup.tar.gz -C /tmp
-
-sudo find /tmp/backup/postgres > /dev/null
-echo "[TEST:PASS] Backup contains files at expected location"
-
-for file in $(sudo find /tmp/backup/postgres); do
-  if [ "$(sudo stat -c '%u:%g' $file)" != "70:70" ]; then
-    echo "[TEST:FAIL] Unexpected file ownership for $file: $(sudo stat -c '%u:%g' $file)"
-    exit 1
-  fi
-done
-echo "[TEST:PASS] All files and directories in backup preserved their ownership."
-
-docker-compose down --volumes

test/swarm/docker-compose.yml

@@ -18,8 +18,8 @@ services:
     volumes:
       - backup_data:/data
 
-  backup:
+  backup: &default_backup_service
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    image: offen/docker-volume-backup:${TEST_VERSION}
     depends_on:
       - minio
     deploy:
@@ -43,15 +43,13 @@ services:
     image: offen/offen:latest
     labels:
       - docker-volume-backup.stop-during-backup=true
-    healthcheck:
-      disable: true
     deploy:
       replicas: 2
       restart_policy:
         condition: on-failure
 
   pg:
-    image: postgres:14-alpine
+    image: postgres:12.2-alpine
     environment:
       POSTGRES_PASSWORD: example
     labels:

test/swarm/run.sh

@@ -23,13 +23,14 @@ docker run --rm -it \
 
 echo "[TEST:PASS] Found relevant files in untared backup."
 
-sleep 5
 if [ "$(docker ps -q | wc -l)" != "5" ]; then
   echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
   docker ps -a
   exit 1
 fi
+
 echo "[TEST:PASS] All containers running post backup."
 
 docker stack rm test_stack
+
 docker swarm leave --force