Replace Gzip with PGzip (#266)

* Replace Gzip with optimized PGzip. Add concurrency option.

* Add shortened timeout for 'dc down' too.

* Add NaturalNumberZero to allow zero.

* Add test for concurrency=0

* Rename to GZIP_PARALLELISM

* Fix block size. Fix compression level. Fix CI.

* Refactor compression writer fetching. Renamed WholeNumber

.circleci/config.yml

@@ -1,70 +0,0 @@
-version: 2.1
-
-jobs:
-  canary:
-    machine:
-      image: ubuntu-1604:202007-01
-    working_directory: ~/docker-volume-backup
-    steps:
-      - checkout
-      - run:
-          name: Build
-          command: |
-            docker build . -t offen/docker-volume-backup:canary
-      - run:
-          name: Install gnupg
-          command: |
-            sudo apt-get install -y gnupg
-      - run:
-          name: Run tests
-          working_directory: ~/docker-volume-backup/test
-          command: |
-            ./test.sh canary
-
-  build:
-    docker:
-      - image: cimg/base:2020.06
-        environment:
-          DOCKER_BUILDKIT: '1'
-          DOCKER_CLI_EXPERIMENTAL: enabled
-    working_directory: ~/docker-volume-backup
-    steps:
-      - checkout
-      - setup_remote_docker:
-          version: 20.10.6
-      - docker/install-docker-credential-helper
-      - docker/configure-docker-credentials-store
-      - run:
-          name: Push to Docker Hub
-          command: |
-            echo "$DOCKER_ACCESSTOKEN" | docker login --username offen --password-stdin
-            # This is required for building ARM: https://gitlab.alpinelinux.org/alpine/aports/-/issues/12406
-            docker run --rm --privileged linuxkit/binfmt:v0.8
-            docker context create docker-volume-backup
-            docker buildx create docker-volume-backup --name docker-volume-backup --use
-            docker buildx inspect --bootstrap
-            tag_args="-t offen/docker-volume-backup:$CIRCLE_TAG"
-            if [[ "$CIRCLE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-              # prerelease tags like `v2.0.0-alpha.1` should not be released as `latest`
-              tag_args="$tag_args -t offen/docker-volume-backup:latest"
-            fi
-            docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 \
-               $tag_args . --push
-
-workflows:
-  version: 2
-  docker_image:
-    jobs:
-      - canary:
-          filters:
-            tags:
-              ignore: /^v.*/
-      - build:
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: /^v.*/
-
-orbs:
-  docker: circleci/docker@1.0.1

.dockerignore

@@ -1 +1,7 @@
 test
+.github
+.circleci
+docs
+.editorconfig
+LICENSE
+README.md

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+github: offen
+patreon: offen
+

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+<!--
+A clear and concise description of what the bug is.
+-->
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. ...
+2. ...
+3. ...
+
+**Expected behavior**
+<!--
+A clear and concise description of what you expected to happen.
+-->
+
+**Version (please complete the following information):**
+ - Image Version: <!-- e.g. v2.21.0 -->
+ - Docker Version: <!-- e.g. 20.10.17 -->
+ - Docker Compose Version (if applicable): <!-- e.g. 1.29.2 -->
+
+**Additional context**
+<!--
+Add any other context about the problem here.
+-->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,28 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+<!--
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+-->
+
+**Describe the solution you'd like**
+<!--
+A clear and concise description of what you want to happen.
+-->
+
+**Describe alternatives you've considered**
+<!--
+A clear and concise description of any alternative solutions or features you've considered.
+-->
+
+**Additional context**
+<!--
+Add any other context or screenshots about the feature request here.
+-->

.github/ISSUE_TEMPLATE/support_request.md

@@ -0,0 +1,28 @@
+---
+name: Support request
+about: Ask for help
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**What are you trying to do?**
+<!--
+A clear and concise description of what you are trying to do, but cannot get working.
+-->
+
+**What is your current configuration?**
+<!--
+Add the full configuration you are using. Please redact out any real-world credentials.
+-->
+
+**Log output**
+<!--
+Provide the full log output of your setup.
+-->
+
+**Additional context**
+<!--
+Add any other context or screenshots about the support request here.
+-->

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly

.github/workflows/golangci-lint.yml

@@ -0,0 +1,54 @@
+name: Run Linters
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+          cache: false
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Require: The version of golangci-lint to use.
+          # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
+          # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
+          version: v1.54
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          #
+          # Note: By default, the `.golangci.yml` file should be at the root of the repository.
+          # The location of the configuration file can be changed by using `--config=`
+          # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true, then all caching functionality will be completely disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true
+
+          # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
+          # install-mode: "goinstall"

.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: Release Docker Image
+
+on:
+  push:
+    tags: v**
+
+jobs:
+  push_to_registries:
+    name: Push Docker image to multiple registries
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker tags
+        id: meta
+        run: |
+          version_tag="${{github.ref_name}}"
+          tags=($version_tag)
+          if [[ "$version_tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            # prerelease tags like `v2.0.0-alpha.1` should not be released as `latest` nor `v2`
+            tags+=("latest")
+            tags+=($(echo "$version_tag" | cut -d. -f1))
+          fi
+          releases=""
+          for tag in "${tags[@]}"; do
+            releases="${releases:+$releases,}offen/docker-volume-backup:$tag,ghcr.io/offen/docker-volume-backup:$tag"
+          done
+          echo "releases=$releases" >> "$GITHUB_OUTPUT"
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          tags: ${{ steps.meta.outputs.releases }}

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: Run Integration Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Run Tests
+        working-directory: ./test
+        run: |
+          BUILD_IMAGE=1 ./test.sh

.golangci.yml

@@ -0,0 +1,8 @@
+linters:
+  # Enable specific linter
+  # https://golangci-lint.run/usage/linters/#enabled-by-default
+  enable:
+    - staticcheck
+    - govet
+output:
+  format: github-actions

Dockerfile

@@ -1,23 +1,21 @@
 # Copyright 2021 - Offen Authors <hioffen@posteo.de>
 # SPDX-License-Identifier: MPL-2.0
 
-FROM golang:1.17-alpine as builder
+FROM golang:1.21-alpine as builder
 
 WORKDIR /app
-COPY go.mod go.sum ./
+COPY . .
 RUN go mod download
-COPY cmd/backup/main.go ./cmd/backup/main.go
+WORKDIR /app/cmd/backup
-RUN go build -o backup cmd/backup/main.go
+RUN go build -o backup .
 
-FROM alpine:3.14
+FROM alpine:3.18
 
 WORKDIR /root
 
-RUN apk add --update ca-certificates
+RUN apk add --no-cache ca-certificates
 
-COPY --from=builder /app/backup /usr/bin/backup
+COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
-
+COPY --chmod=755 ./entrypoint.sh /root/
-COPY ./entrypoint.sh /root/
-RUN chmod +x entrypoint.sh
 
 ENTRYPOINT ["/root/entrypoint.sh"]

cmd/backup/archive.go

@@ -0,0 +1,168 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+// Portions of this file are taken from package `targz`, Copyright (c) 2014 Fredrik Wallgren
+// Licensed under the MIT License: https://github.com/walle/targz/blob/57fe4206da5abf7dd3901b4af3891ec2f08c7b08/LICENSE
+
+package main
+
+import (
+	"archive/tar"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/klauspost/pgzip"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+func createArchive(files []string, inputFilePath, outputFilePath string, compression string, compressionConcurrency int) error {
+	inputFilePath = stripTrailingSlashes(inputFilePath)
+	inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
+	if err != nil {
+		return fmt.Errorf("createArchive: error transposing given file paths: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(outputFilePath), 0755); err != nil {
+		return fmt.Errorf("createArchive: error creating output file path: %w", err)
+	}
+
+	if err := compress(files, outputFilePath, filepath.Dir(inputFilePath), compression, compressionConcurrency); err != nil {
+		return fmt.Errorf("createArchive: error creating archive: %w", err)
+	}
+
+	return nil
+}
+
+func stripTrailingSlashes(path string) string {
+	if len(path) > 0 && path[len(path)-1] == '/' {
+		path = path[0 : len(path)-1]
+	}
+
+	return path
+}
+
+func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error) {
+	inputFilePath, err := filepath.Abs(inputFilePath)
+	if err == nil {
+		outputFilePath, err = filepath.Abs(outputFilePath)
+	}
+
+	return inputFilePath, outputFilePath, err
+}
+
+func compress(paths []string, outFilePath, subPath string, algo string, concurrency int) error {
+	file, err := os.Create(outFilePath)
+	if err != nil {
+		return fmt.Errorf("compress: error creating out file: %w", err)
+	}
+
+	prefix := path.Dir(outFilePath)
+	compressWriter, err := getCompressionWriter(file, algo, concurrency)
+	if err != nil {
+		return fmt.Errorf("compress: error getting compression writer: %w", err)
+	}
+	tarWriter := tar.NewWriter(compressWriter)
+
+	for _, p := range paths {
+		if err := writeTarball(p, tarWriter, prefix); err != nil {
+			return fmt.Errorf("compress: error writing %s to archive: %w", p, err)
+		}
+	}
+
+	err = tarWriter.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing tar writer: %w", err)
+	}
+
+	err = compressWriter.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing compression writer: %w", err)
+	}
+
+	err = file.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing file: %w", err)
+	}
+
+	return nil
+}
+
+func getCompressionWriter(file *os.File, algo string, concurrency int) (io.WriteCloser, error) {
+	switch algo {
+	case "gz":
+		w, err := pgzip.NewWriterLevel(file, 5)
+		if err != nil {
+			return nil, fmt.Errorf("getCompressionWriter: gzip error: %w", err)
+		}
+
+		if concurrency == 0 {
+			concurrency = runtime.GOMAXPROCS(0)
+		}
+
+		if err := w.SetConcurrency(1<<20, concurrency); err != nil {
+			return nil, fmt.Errorf("getCompressionWriter: error setting concurrency: %w", err)
+		}
+
+		return w, nil
+	case "zst":
+		compressWriter, err := zstd.NewWriter(file)
+		if err != nil {
+			return nil, fmt.Errorf("getCompressionWriter: zstd error: %w", err)
+		}
+		return compressWriter, nil
+	default:
+		return nil, fmt.Errorf("getCompressionWriter: unsupported compression algorithm: %s", algo)
+	}
+}
+
+func writeTarball(path string, tarWriter *tar.Writer, prefix string) error {
+	fileInfo, err := os.Lstat(path)
+	if err != nil {
+		return fmt.Errorf("writeTarball: error getting file infor for %s: %w", path, err)
+	}
+
+	if fileInfo.Mode()&os.ModeSocket == os.ModeSocket {
+		return nil
+	}
+
+	var link string
+	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
+		var err error
+		if link, err = os.Readlink(path); err != nil {
+			return fmt.Errorf("writeTarball: error resolving symlink %s: %w", path, err)
+		}
+	}
+
+	header, err := tar.FileInfoHeader(fileInfo, link)
+	if err != nil {
+		return fmt.Errorf("writeTarball: error getting file info header: %w", err)
+	}
+	header.Name = strings.TrimPrefix(path, prefix)
+
+	err = tarWriter.WriteHeader(header)
+	if err != nil {
+		return fmt.Errorf("writeTarball: error writing file info header: %w", err)
+	}
+
+	if !fileInfo.Mode().IsRegular() {
+		return nil
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("writeTarball: error opening %s: %w", path, err)
+	}
+	defer file.Close()
+
+	_, err = io.Copy(tarWriter, file)
+	if err != nil {
+		return fmt.Errorf("writeTarball: error copying %s to tar writer: %w", path, err)
+	}
+
+	return nil
+}

cmd/backup/config.go

@@ -0,0 +1,171 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+// Config holds all configuration values that are expected to be set
+// by users.
+type Config struct {
+	AwsS3BucketName               string          `split_words:"true"`
+	AwsS3Path                     string          `split_words:"true"`
+	AwsEndpoint                   string          `split_words:"true" default:"s3.amazonaws.com"`
+	AwsEndpointProto              string          `split_words:"true" default:"https"`
+	AwsEndpointInsecure           bool            `split_words:"true"`
+	AwsEndpointCACert             CertDecoder     `envconfig:"AWS_ENDPOINT_CA_CERT"`
+	AwsStorageClass               string          `split_words:"true"`
+	AwsAccessKeyID                string          `envconfig:"AWS_ACCESS_KEY_ID"`
+	AwsSecretAccessKey            string          `split_words:"true"`
+	AwsIamRoleEndpoint            string          `split_words:"true"`
+	AwsPartSize                   int64           `split_words:"true"`
+	BackupCompression             CompressionType `split_words:"true" default:"gz"`
+	GzipParallelism               WholeNumber     `split_words:"true" default:"1"`
+	BackupSources                 string          `split_words:"true" default:"/backup"`
+	BackupFilename                string          `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"`
+	BackupFilenameExpand          bool            `split_words:"true"`
+	BackupLatestSymlink           string          `split_words:"true"`
+	BackupArchive                 string          `split_words:"true" default:"/archive"`
+	BackupRetentionDays           int32           `split_words:"true" default:"-1"`
+	BackupPruningLeeway           time.Duration   `split_words:"true" default:"1m"`
+	BackupPruningPrefix           string          `split_words:"true"`
+	BackupStopContainerLabel      string          `split_words:"true" default:"true"`
+	BackupFromSnapshot            bool            `split_words:"true"`
+	BackupExcludeRegexp           RegexpDecoder   `split_words:"true"`
+	BackupSkipBackendsFromPrune   []string        `split_words:"true"`
+	GpgPassphrase                 string          `split_words:"true"`
+	NotificationURLs              []string        `envconfig:"NOTIFICATION_URLS"`
+	NotificationLevel             string          `split_words:"true" default:"error"`
+	EmailNotificationRecipient    string          `split_words:"true"`
+	EmailNotificationSender       string          `split_words:"true" default:"noreply@nohost"`
+	EmailSMTPHost                 string          `envconfig:"EMAIL_SMTP_HOST"`
+	EmailSMTPPort                 int             `envconfig:"EMAIL_SMTP_PORT" default:"587"`
+	EmailSMTPUsername             string          `envconfig:"EMAIL_SMTP_USERNAME"`
+	EmailSMTPPassword             string          `envconfig:"EMAIL_SMTP_PASSWORD"`
+	WebdavUrl                     string          `split_words:"true"`
+	WebdavUrlInsecure             bool            `split_words:"true"`
+	WebdavPath                    string          `split_words:"true" default:"/"`
+	WebdavUsername                string          `split_words:"true"`
+	WebdavPassword                string          `split_words:"true"`
+	SSHHostName                   string          `split_words:"true"`
+	SSHPort                       string          `split_words:"true" default:"22"`
+	SSHUser                       string          `split_words:"true"`
+	SSHPassword                   string          `split_words:"true"`
+	SSHIdentityFile               string          `split_words:"true" default:"/root/.ssh/id_rsa"`
+	SSHIdentityPassphrase         string          `split_words:"true"`
+	SSHRemotePath                 string          `split_words:"true"`
+	ExecLabel                     string          `split_words:"true"`
+	ExecForwardOutput             bool            `split_words:"true"`
+	LockTimeout                   time.Duration   `split_words:"true" default:"60m"`
+	AzureStorageAccountName       string          `split_words:"true"`
+	AzureStoragePrimaryAccountKey string          `split_words:"true"`
+	AzureStorageContainerName     string          `split_words:"true"`
+	AzureStoragePath              string          `split_words:"true"`
+	AzureStorageEndpoint          string          `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
+	DropboxEndpoint               string          `split_words:"true" default:"https://api.dropbox.com/"`
+	DropboxOAuth2Endpoint         string          `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"`
+	DropboxRefreshToken           string          `split_words:"true"`
+	DropboxAppKey                 string          `split_words:"true"`
+	DropboxAppSecret              string          `split_words:"true"`
+	DropboxRemotePath             string          `split_words:"true"`
+	DropboxConcurrencyLevel       NaturalNumber   `split_words:"true" default:"6"`
+}
+
+type CompressionType string
+
+func (c *CompressionType) Decode(v string) error {
+	switch v {
+	case "gz", "zst":
+		*c = CompressionType(v)
+		return nil
+	default:
+		return fmt.Errorf("config: error decoding compression type %s", v)
+	}
+}
+
+func (c *CompressionType) String() string {
+	return string(*c)
+}
+
+type CertDecoder struct {
+	Cert *x509.Certificate
+}
+
+func (c *CertDecoder) Decode(v string) error {
+	if v == "" {
+		return nil
+	}
+	content, err := os.ReadFile(v)
+	if err != nil {
+		content = []byte(v)
+	}
+	block, _ := pem.Decode(content)
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return fmt.Errorf("config: error parsing certificate: %w", err)
+	}
+	*c = CertDecoder{Cert: cert}
+	return nil
+}
+
+type RegexpDecoder struct {
+	Re *regexp.Regexp
+}
+
+func (r *RegexpDecoder) Decode(v string) error {
+	if v == "" {
+		return nil
+	}
+	re, err := regexp.Compile(v)
+	if err != nil {
+		return fmt.Errorf("config: error compiling given regexp `%s`: %w", v, err)
+	}
+	*r = RegexpDecoder{Re: re}
+	return nil
+}
+
+// NaturalNumber is a type that can be used to decode a positive, non-zero natural number
+type NaturalNumber int
+
+func (n *NaturalNumber) Decode(v string) error {
+	asInt, err := strconv.Atoi(v)
+	if err != nil {
+		return fmt.Errorf("config: error converting %s to int", v)
+	}
+	if asInt <= 0 {
+		return fmt.Errorf("config: expected a natural number, got %d", asInt)
+	}
+	*n = NaturalNumber(asInt)
+	return nil
+}
+
+func (n *NaturalNumber) Int() int {
+	return int(*n)
+}
+
+// WholeNumber is a type that can be used to decode a positive whole number, including zero
+type WholeNumber int
+
+func (n *WholeNumber) Decode(v string) error {
+	asInt, err := strconv.Atoi(v)
+	if err != nil {
+		return fmt.Errorf("config: error converting %s to int", v)
+	}
+	if asInt < 0 {
+		return fmt.Errorf("config: expected a whole, positive number, including zero. Got %d", asInt)
+	}
+	*n = WholeNumber(asInt)
+	return nil
+}
+
+func (n *WholeNumber) Int() int {
+	return int(*n)
+}

cmd/backup/exec.go

@@ -0,0 +1,200 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+// Portions of this file are taken and adapted from `moby`, Copyright 2012-2017 Docker, Inc.
+// Licensed under the Apache 2.0 License: https://github.com/moby/moby/blob/8e610b2b55bfd1bfa9436ab110d311f5e8a74dcb/LICENSE
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/cosiner/argv"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/pkg/stdcopy"
+	"golang.org/x/sync/errgroup"
+)
+
+func (s *script) exec(containerRef string, command string, user string) ([]byte, []byte, error) {
+	args, _ := argv.Argv(command, nil, nil)
+	commandEnv := []string{
+		fmt.Sprintf("COMMAND_RUNTIME_ARCHIVE_FILEPATH=%s", s.file),
+	}
+	execID, err := s.cli.ContainerExecCreate(context.Background(), containerRef, types.ExecConfig{
+		Cmd:          args[0],
+		AttachStdin:  true,
+		AttachStderr: true,
+		Env:          commandEnv,
+		User:         user,
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error creating container exec: %w", err)
+	}
+
+	resp, err := s.cli.ContainerExecAttach(context.Background(), execID.ID, types.ExecStartCheck{})
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error attaching container exec: %w", err)
+	}
+	defer resp.Close()
+
+	var outBuf, errBuf bytes.Buffer
+	outputDone := make(chan error)
+
+	go func() {
+		_, err := stdcopy.StdCopy(&outBuf, &errBuf, resp.Reader)
+		outputDone <- err
+	}()
+
+	if <-outputDone != nil {
+		return nil, nil, fmt.Errorf("exec: error demultiplexing output: %w", err)
+	}
+
+	stdout, err := io.ReadAll(&outBuf)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error reading stdout: %w", err)
+	}
+	stderr, err := io.ReadAll(&errBuf)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error reading stderr: %w", err)
+	}
+
+	res, err := s.cli.ContainerExecInspect(context.Background(), execID.ID)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error inspecting container exec: %w", err)
+	}
+
+	if res.ExitCode > 0 {
+		return stdout, stderr, fmt.Errorf("exec: running command exited %d", res.ExitCode)
+	}
+
+	return stdout, stderr, nil
+}
+
+func (s *script) runLabeledCommands(label string) error {
+	f := []filters.KeyValuePair{
+		{Key: "label", Value: label},
+	}
+	if s.c.ExecLabel != "" {
+		f = append(f, filters.KeyValuePair{
+			Key:   "label",
+			Value: fmt.Sprintf("docker-volume-backup.exec-label=%s", s.c.ExecLabel),
+		})
+	}
+	containersWithCommand, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Filters: filters.NewArgs(f...),
+	})
+	if err != nil {
+		return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
+	}
+
+	var hasDeprecatedContainers bool
+	if label == "docker-volume-backup.archive-pre" {
+		f[0] = filters.KeyValuePair{
+			Key:   "label",
+			Value: "docker-volume-backup.exec-pre",
+		}
+		deprecatedContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+			Filters: filters.NewArgs(f...),
+		})
+		if err != nil {
+			return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
+		}
+		if len(deprecatedContainers) != 0 {
+			hasDeprecatedContainers = true
+			containersWithCommand = append(containersWithCommand, deprecatedContainers...)
+		}
+	}
+
+	if label == "docker-volume-backup.archive-post" {
+		f[0] = filters.KeyValuePair{
+			Key:   "label",
+			Value: "docker-volume-backup.exec-post",
+		}
+		deprecatedContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+			Filters: filters.NewArgs(f...),
+		})
+		if err != nil {
+			return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
+		}
+		if len(deprecatedContainers) != 0 {
+			hasDeprecatedContainers = true
+			containersWithCommand = append(containersWithCommand, deprecatedContainers...)
+		}
+	}
+
+	if len(containersWithCommand) == 0 {
+		return nil
+	}
+
+	if hasDeprecatedContainers {
+		s.logger.Warn(
+			"Using `docker-volume-backup.exec-pre` and `docker-volume-backup.exec-post` labels has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use other `-pre` and `-post` labels instead. Refer to the README for an upgrade guide.",
+		)
+	}
+
+	g := new(errgroup.Group)
+
+	for _, container := range containersWithCommand {
+		c := container
+		g.Go(func() error {
+			cmd, ok := c.Labels[label]
+			if !ok && label == "docker-volume-backup.archive-pre" {
+				cmd = c.Labels["docker-volume-backup.exec-pre"]
+			} else if !ok && label == "docker-volume-backup.archive-post" {
+				cmd = c.Labels["docker-volume-backup.exec-post"]
+			}
+
+			userLabelName := fmt.Sprintf("%s.user", label)
+			user := c.Labels[userLabelName]
+
+			s.logger.Info(fmt.Sprintf("Running %s command %s for container %s", label, cmd, strings.TrimPrefix(c.Names[0], "/")))
+			stdout, stderr, err := s.exec(c.ID, cmd, user)
+			if s.c.ExecForwardOutput {
+				os.Stderr.Write(stderr)
+				os.Stdout.Write(stdout)
+			}
+			if err != nil {
+				return fmt.Errorf("runLabeledCommands: error executing command: %w", err)
+			}
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		return fmt.Errorf("runLabeledCommands: error from errgroup: %w", err)
+	}
+	return nil
+}
+
+type lifecyclePhase string
+
+const (
+	lifecyclePhaseArchive lifecyclePhase = "archive"
+	lifecyclePhaseProcess lifecyclePhase = "process"
+	lifecyclePhaseCopy    lifecyclePhase = "copy"
+	lifecyclePhasePrune   lifecyclePhase = "prune"
+)
+
+func (s *script) withLabeledCommands(step lifecyclePhase, cb func() error) func() error {
+	if s.cli == nil {
+		return cb
+	}
+	return func() error {
+		if err := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil {
+			return fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err)
+		}
+		defer func() {
+			s.must(s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step)))
+		}()
+		return cb()
+	}
+}

cmd/backup/hooks.go

@@ -0,0 +1,57 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+)
+
+// hook contains a queued action that can be trigger them when the script
+// reaches a certain point (e.g. unsuccessful backup)
+type hook struct {
+	level  hookLevel
+	action func(err error) error
+}
+
+type hookLevel int
+
+const (
+	hookLevelPlumbing hookLevel = iota
+	hookLevelError
+	hookLevelInfo
+)
+
+var hookLevels = map[string]hookLevel{
+	"info":  hookLevelInfo,
+	"error": hookLevelError,
+}
+
+// registerHook adds the given action at the given level.
+func (s *script) registerHook(level hookLevel, action func(err error) error) {
+	s.hooks = append(s.hooks, hook{level, action})
+}
+
+// runHooks runs all hooks that have been registered using the
+// given levels in the defined ordering. In case executing a hook returns an
+// error, the following hooks will still be run before the function returns.
+func (s *script) runHooks(err error) error {
+	sort.SliceStable(s.hooks, func(i, j int) bool {
+		return s.hooks[i].level < s.hooks[j].level
+	})
+	var actionErrors []error
+	for _, hook := range s.hooks {
+		if hook.level > s.hookLevel {
+			continue
+		}
+		if actionErr := hook.action(err); actionErr != nil {
+			actionErrors = append(actionErrors, fmt.Errorf("runHooks: error running hook: %w", actionErr))
+		}
+	}
+	if len(actionErrors) != 0 {
+		return errors.Join(actionErrors...)
+	}
+	return nil
+}

cmd/backup/lock.go

@@ -0,0 +1,60 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/gofrs/flock"
+)
+
+// lock opens a lockfile at the given location, keeping it locked until the
+// caller invokes the returned release func. In case the lock is currently blocked
+// by another execution, it will repeatedly retry until the lock is available
+// or the given timeout is exceeded.
+func (s *script) lock(lockfile string) (func() error, error) {
+	start := time.Now()
+	defer func() {
+		s.stats.LockedTime = time.Since(start)
+	}()
+
+	retry := time.NewTicker(5 * time.Second)
+	defer retry.Stop()
+	deadline := time.NewTimer(s.c.LockTimeout)
+	defer deadline.Stop()
+
+	fileLock := flock.New(lockfile)
+
+	for {
+		acquired, err := fileLock.TryLock()
+		if err != nil {
+			return noop, fmt.Errorf("lock: error trying to lock: %w", err)
+		}
+		if acquired {
+			if s.encounteredLock {
+				s.logger.Info("Acquired exclusive lock on subsequent attempt, ready to continue.")
+			}
+			return fileLock.Unlock, nil
+		}
+
+		if !s.encounteredLock {
+			s.logger.Info(
+				fmt.Sprintf(
+					"Exclusive lock was not available on first attempt. Will retry until it becomes available or the timeout of %s is exceeded.",
+					s.c.LockTimeout,
+				),
+			)
+			s.encounteredLock = true
+		}
+
+		select {
+		case <-retry.C:
+			continue
+		case <-deadline.C:
+			return noop, errors.New("lock: timed out waiting for lockfile to become available")
+		}
+	}
+}

cmd/backup/main.go

@@ -1,53 +1,30 @@
-// Copyright 2021 - Offen Authors <hioffen@posteo.de>
+// Copyright 2021-2022 - Offen Authors <hioffen@posteo.de>
 // SPDX-License-Identifier: MPL-2.0
 
 package main
 
 import (
-	"bytes"
-	"context"
-	"errors"
 	"fmt"
-	"io"
 	"os"
-	"path"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/containrrr/shoutrrr"
-	"github.com/containrrr/shoutrrr/pkg/router"
-	sTypes "github.com/containrrr/shoutrrr/pkg/types"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/api/types/swarm"
-	"github.com/docker/docker/client"
-	"github.com/gofrs/flock"
-	"github.com/kelseyhightower/envconfig"
-	"github.com/leekchan/timeutil"
-	"github.com/m90/targz"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/otiai10/copy"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/crypto/openpgp"
 )
 
 func main() {
-	unlock := lock("/var/lock/dockervolumebackup.lock")
-	defer unlock()
-
 	s, err := newScript()
 	if err != nil {
 		panic(err)
 	}
 
+	unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
+	defer s.must(unlock())
+	s.must(err)
+
 	defer func() {
 		if pArg := recover(); pArg != nil {
 			if err, ok := pArg.(error); ok {
 				if hookErr := s.runHooks(err); hookErr != nil {
-					s.logger.Errorf("An error occurred calling the registered hooks: %s", hookErr)
+					s.logger.Error(
+						fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr),
+					)
 				}
 				os.Exit(1)
 			}
@@ -55,16 +32,18 @@ func main() {
 		}
 
 		if err := s.runHooks(nil); err != nil {
-			s.logger.Errorf(
+			s.logger.Error(
+				fmt.Sprintf(
 					"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
 					err,
+				),
 			)
 			os.Exit(1)
 		}
 		s.logger.Info("Finished running backup tasks.")
 	}()
 
-	s.must(func() error {
+	s.must(s.withLabeledCommands(lifecyclePhaseArchive, func() error {
 		restartContainers, err := s.stopContainers()
 		// The mechanism for restarting containers is not using hooks as it
 		// should happen as soon as possible (i.e. before uploading backups or
@@ -75,758 +54,10 @@ func main() {
 		if err != nil {
 			return err
 		}
-		return s.takeBackup()
+		return s.createArchive()
-	}())
+	})())
 
-	s.must(s.encryptBackup())
+	s.must(s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)())
-	s.must(s.copyBackup())
+	s.must(s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)())
-	s.must(s.pruneOldBackups())
+	s.must(s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)())
-}
-
-// script holds all the stateful information required to orchestrate a
-// single backup run.
-type script struct {
-	cli       *client.Client
-	mc        *minio.Client
-	logger    *logrus.Logger
-	sender    *router.ServiceRouter
-	hooks     []hook
-	hookLevel hookLevel
-
-	start  time.Time
-	file   string
-	output *bytes.Buffer
-
-	c *config
-}
-
-type config struct {
-	BackupSources              string        `split_words:"true" default:"/backup"`
-	BackupFilename             string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
-	BackupFilenameExpand       bool          `split_words:"true"`
-	BackupLatestSymlink        string        `split_words:"true"`
-	BackupArchive              string        `split_words:"true" default:"/archive"`
-	BackupRetentionDays        int32         `split_words:"true" default:"-1"`
-	BackupPruningLeeway        time.Duration `split_words:"true" default:"1m"`
-	BackupPruningPrefix        string        `split_words:"true"`
-	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
-	BackupFromSnapshot         bool          `split_words:"true"`
-	AwsS3BucketName            string        `split_words:"true"`
-	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
-	AwsEndpointProto           string        `split_words:"true" default:"https"`
-	AwsEndpointInsecure        bool          `split_words:"true"`
-	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
-	AwsSecretAccessKey         string        `split_words:"true"`
-	AwsIamRoleEndpoint         string        `split_words:"true"`
-	GpgPassphrase              string        `split_words:"true"`
-	NotificationURLs           []string      `envconfig:"NOTIFICATION_URLS"`
-	NotificationLevel          string        `split_words:"true" default:"error"`
-	EmailNotificationRecipient string        `split_words:"true"`
-	EmailNotificationSender    string        `split_words:"true" default:"noreply@nohost"`
-	EmailSMTPHost              string        `envconfig:"EMAIL_SMTP_HOST"`
-	EmailSMTPPort              int           `envconfig:"EMAIL_SMTP_PORT" default:"587"`
-	EmailSMTPUsername          string        `envconfig:"EMAIL_SMTP_USERNAME"`
-	EmailSMTPPassword          string        `envconfig:"EMAIL_SMTP_PASSWORD"`
-}
-
-var msgBackupFailed = "backup run failed"
-
-// newScript creates all resources needed for the script to perform actions against
-// remote resources like the Docker engine or remote storage locations. All
-// reading from env vars or other configuration sources is expected to happen
-// in this method.
-func newScript() (*script, error) {
-	stdOut, logBuffer := buffer(os.Stdout)
-	s := &script{
-		c: &config{},
-		logger: &logrus.Logger{
-			Out:       stdOut,
-			Formatter: new(logrus.TextFormatter),
-			Hooks:     make(logrus.LevelHooks),
-			Level:     logrus.InfoLevel,
-		},
-		start:  time.Now(),
-		output: logBuffer,
-	}
-
-	if err := envconfig.Process("", s.c); err != nil {
-		return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
-	}
-
-	s.file = path.Join("/tmp", s.c.BackupFilename)
-	if s.c.BackupFilenameExpand {
-		s.file = os.ExpandEnv(s.file)
-		s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
-		s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix)
-	}
-	s.file = timeutil.Strftime(&s.start, s.file)
-
-	_, err := os.Stat("/var/run/docker.sock")
-	if !os.IsNotExist(err) {
-		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-		if err != nil {
-			return nil, fmt.Errorf("newScript: failed to create docker client")
-		}
-		s.cli = cli
-	}
-
-	if s.c.AwsS3BucketName != "" {
-		var creds *credentials.Credentials
-		if s.c.AwsAccessKeyID != "" && s.c.AwsSecretAccessKey != "" {
-			creds = credentials.NewStaticV4(
-				s.c.AwsAccessKeyID,
-				s.c.AwsSecretAccessKey,
-				"",
-			)
-		} else if s.c.AwsIamRoleEndpoint != "" {
-			creds = credentials.NewIAM(s.c.AwsIamRoleEndpoint)
-		} else {
-			return nil, errors.New("newScript: AWS_S3_BUCKET_NAME is defined, but no credentials were provided")
-		}
-
-		options := minio.Options{
-			Creds:  creds,
-			Secure: s.c.AwsEndpointProto == "https",
-		}
-
-		if s.c.AwsEndpointInsecure {
-			if !options.Secure {
-				return nil, errors.New("newScript: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
-			}
-
-			transport, err := minio.DefaultTransport(true)
-			if err != nil {
-				return nil, fmt.Errorf("newScript: failed to create default minio transport")
-			}
-			transport.TLSClientConfig.InsecureSkipVerify = true
-			options.Transport = transport
-		}
-
-		mc, err := minio.New(s.c.AwsEndpoint, &options)
-		if err != nil {
-			return nil, fmt.Errorf("newScript: error setting up minio client: %w", err)
-		}
-		s.mc = mc
-	}
-
-	if s.c.EmailNotificationRecipient != "" {
-		emailURL := fmt.Sprintf(
-			"smtp://%s:%s@%s:%d/?from=%s&to=%s",
-			s.c.EmailSMTPUsername,
-			s.c.EmailSMTPPassword,
-			s.c.EmailSMTPHost,
-			s.c.EmailSMTPPort,
-			s.c.EmailNotificationSender,
-			s.c.EmailNotificationRecipient,
-		)
-		s.c.NotificationURLs = append(s.c.NotificationURLs, emailURL)
-		s.logger.Warn(
-			"Using EMAIL_* keys for providing notification configuration has been deprecated and will be removed in the next major version.",
-		)
-		s.logger.Warn(
-			"Please use NOTIFICATION_URLS instead. Refer to the README for an upgrade guide.",
-		)
-	}
-
-	hookLevel, ok := hookLevels[s.c.NotificationLevel]
-	if !ok {
-		return nil, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel)
-	}
-	s.hookLevel = hookLevel
-
-	if len(s.c.NotificationURLs) > 0 {
-		sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...)
-		if senderErr != nil {
-			return nil, fmt.Errorf("newScript: error creating sender: %w", senderErr)
-		}
-		s.sender = sender
-		// To prevent duplicate notifications, ensure the regsistered callbacks
-		// run mutually exclusive.
-		s.registerHook(hookLevelError, func(err error) error {
-			if err == nil {
-				return nil
-			}
-			return s.notifyFailure(err)
-		})
-		s.registerHook(hookLevelInfo, func(err error) error {
-			if err != nil {
-				return nil
-			}
-			return s.notifySuccess()
-		})
-	}
-
-	return s, nil
-}
-
-var noop = func() error { return nil }
-
-// registerHook adds the given action at the given level.
-func (s *script) registerHook(level hookLevel, action func(err error) error) {
-	s.hooks = append(s.hooks, hook{level, action})
-}
-
-// notifyFailure sends a notification about a failed backup run
-func (s *script) notifyFailure(err error) error {
-	body := fmt.Sprintf(
-		"Running docker-volume-backup failed with error: %s\n\nLog output of the failed run was:\n\n%s\n", err, s.output.String(),
-	)
-	title := fmt.Sprintf("Failure running docker-volume-backup at %s", s.start.Format(time.RFC3339))
-	if err := s.sendNotification(title, body); err != nil {
-		return fmt.Errorf("notifyFailure: error notifying: %w", err)
-	}
-	return nil
-}
-
-// notifyFailure sends a notification about a successful backup run
-func (s *script) notifySuccess() error {
-	title := fmt.Sprintf("Success running docker-volume-backup at %s", s.start.Format(time.RFC3339))
-	body := fmt.Sprintf(
-		"Running docker-volume-backup succeeded.\n\nLog output was:\n\n%s\n", s.output.String(),
-	)
-	if err := s.sendNotification(title, body); err != nil {
-		return fmt.Errorf("notifySuccess: error notifying: %w", err)
-	}
-	return nil
-}
-
-// sendNotification sends a notification to all configured third party services
-func (s *script) sendNotification(title, body string) error {
-	var errs []error
-	for _, result := range s.sender.Send(body, &sTypes.Params{"title": title}) {
-		if result != nil {
-			errs = append(errs, result)
-		}
-	}
-	if len(errs) != 0 {
-		return fmt.Errorf("sendNotification: error sending message: %w", join(errs...))
-	}
-	return nil
-}
-
-// stopContainers stops all Docker containers that are marked as to being
-// stopped during the backup and returns a function that can be called to
-// restart everything that has been stopped.
-func (s *script) stopContainers() (func() error, error) {
-	if s.cli == nil {
-		return noop, nil
-	}
-
-	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Quiet: true,
-	})
-	if err != nil {
-		return noop, fmt.Errorf("stopContainersAndRun: error querying for containers: %w", err)
-	}
-
-	containerLabel := fmt.Sprintf(
-		"docker-volume-backup.stop-during-backup=%s",
-		s.c.BackupStopContainerLabel,
-	)
-	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Quiet: true,
-		Filters: filters.NewArgs(filters.KeyValuePair{
-			Key:   "label",
-			Value: containerLabel,
-		}),
-	})
-
-	if err != nil {
-		return noop, fmt.Errorf("stopContainersAndRun: error querying for containers to stop: %w", err)
-	}
-
-	if len(containersToStop) == 0 {
-		return noop, nil
-	}
-
-	s.logger.Infof(
-		"Stopping %d container(s) labeled `%s` out of %d running container(s).",
-		len(containersToStop),
-		containerLabel,
-		len(allContainers),
-	)
-
-	var stoppedContainers []types.Container
-	var stopErrors []error
-	for _, container := range containersToStop {
-		if err := s.cli.ContainerStop(context.Background(), container.ID, nil); err != nil {
-			stopErrors = append(stopErrors, err)
-		} else {
-			stoppedContainers = append(stoppedContainers, container)
-		}
-	}
-
-	var stopError error
-	if len(stopErrors) != 0 {
-		stopError = fmt.Errorf(
-			"stopContainersAndRun: %d error(s) stopping containers: %w",
-			len(stopErrors),
-			join(stopErrors...),
-		)
-	}
-
-	return func() error {
-		servicesRequiringUpdate := map[string]struct{}{}
-
-		var restartErrors []error
-		for _, container := range stoppedContainers {
-			if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok {
-				servicesRequiringUpdate[swarmServiceName] = struct{}{}
-				continue
-			}
-			if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil {
-				restartErrors = append(restartErrors, err)
-			}
-		}
-
-		if len(servicesRequiringUpdate) != 0 {
-			services, _ := s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
-			for serviceName := range servicesRequiringUpdate {
-				var serviceMatch swarm.Service
-				for _, service := range services {
-					if service.Spec.Name == serviceName {
-						serviceMatch = service
-						break
-					}
-				}
-				if serviceMatch.ID == "" {
-					return fmt.Errorf("stopContainersAndRun: couldn't find service with name %s", serviceName)
-				}
-				serviceMatch.Spec.TaskTemplate.ForceUpdate = 1
-				if _, err := s.cli.ServiceUpdate(
-					context.Background(), serviceMatch.ID,
-					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},
-				); err != nil {
-					restartErrors = append(restartErrors, err)
-				}
-			}
-		}
-
-		if len(restartErrors) != 0 {
-			return fmt.Errorf(
-				"stopContainersAndRun: %d error(s) restarting containers and services: %w",
-				len(restartErrors),
-				join(restartErrors...),
-			)
-		}
-		s.logger.Infof(
-			"Restarted %d container(s) and the matching service(s).",
-			len(stoppedContainers),
-		)
-		return nil
-	}, stopError
-}
-
-// takeBackup creates a tar archive of the configured backup location and
-// saves it to disk.
-func (s *script) takeBackup() error {
-	backupSources := s.c.BackupSources
-
-	if s.c.BackupFromSnapshot {
-		backupSources = filepath.Join("/tmp", s.c.BackupSources)
-		// copy before compressing guard against a situation where backup folder's content are still growing.
-		s.registerHook(hookLevelPlumbing, func(error) error {
-			if err := remove(backupSources); err != nil {
-				return fmt.Errorf("takeBackup: error removing snapshot: %w", err)
-			}
-			s.logger.Infof("Removed snapshot `%s`.", backupSources)
-			return nil
-		})
-		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
-			PreserveTimes: true,
-			PreserveOwner: true,
-		}); err != nil {
-			return fmt.Errorf("takeBackup: error creating snapshot: %w", err)
-		}
-		s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources)
-	}
-
-	tarFile := s.file
-	s.registerHook(hookLevelPlumbing, func(error) error {
-		if err := remove(tarFile); err != nil {
-			return fmt.Errorf("takeBackup: error removing tar file: %w", err)
-		}
-		s.logger.Infof("Removed tar file `%s`.", tarFile)
-		return nil
-	})
-	if err := targz.Compress(backupSources, tarFile); err != nil {
-		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
-	}
-
-	s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, tarFile)
-	return nil
-}
-
-// encryptBackup encrypts the backup file using PGP and the configured passphrase.
-// In case no passphrase is given it returns early, leaving the backup file
-// untouched.
-func (s *script) encryptBackup() error {
-	if s.c.GpgPassphrase == "" {
-		return nil
-	}
-
-	gpgFile := fmt.Sprintf("%s.gpg", s.file)
-	s.registerHook(hookLevelPlumbing, func(error) error {
-		if err := remove(gpgFile); err != nil {
-			return fmt.Errorf("encryptBackup: error removing gpg file: %w", err)
-		}
-		s.logger.Infof("Removed GPG file `%s`.", gpgFile)
-		return nil
-	})
-
-	outFile, err := os.Create(gpgFile)
-	defer outFile.Close()
-	if err != nil {
-		return fmt.Errorf("encryptBackup: error opening out file: %w", err)
-	}
-
-	_, name := path.Split(s.file)
-	dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
-		IsBinary: true,
-		FileName: name,
-	}, nil)
-	defer dst.Close()
-	if err != nil {
-		return fmt.Errorf("encryptBackup: error encrypting backup file: %w", err)
-	}
-
-	src, err := os.Open(s.file)
-	if err != nil {
-		return fmt.Errorf("encryptBackup: error opening backup file `%s`: %w", s.file, err)
-	}
-
-	if _, err := io.Copy(dst, src); err != nil {
-		return fmt.Errorf("encryptBackup: error writing ciphertext to file: %w", err)
-	}
-
-	s.file = gpgFile
-	s.logger.Infof("Encrypted backup using given passphrase, saving as `%s`.", s.file)
-	return nil
-}
-
-// copyBackup makes sure the backup file is copied to both local and remote locations
-// as per the given configuration.
-func (s *script) copyBackup() error {
-	_, name := path.Split(s.file)
-	if s.mc != nil {
-		if _, err := s.mc.FPutObject(context.Background(), s.c.AwsS3BucketName, name, s.file, minio.PutObjectOptions{
-			ContentType: "application/tar+gzip",
-		}); err != nil {
-			return fmt.Errorf("copyBackup: error uploading backup to remote storage: %w", err)
-		}
-		s.logger.Infof("Uploaded a copy of backup `%s` to bucket `%s`.", s.file, s.c.AwsS3BucketName)
-	}
-
-	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
-		if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
-			return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
-		}
-		s.logger.Infof("Stored copy of backup `%s` in local archive `%s`.", s.file, s.c.BackupArchive)
-		if s.c.BackupLatestSymlink != "" {
-			symlink := path.Join(s.c.BackupArchive, s.c.BackupLatestSymlink)
-			if _, err := os.Lstat(symlink); err == nil {
-				os.Remove(symlink)
-			}
-			if err := os.Symlink(name, symlink); err != nil {
-				return fmt.Errorf("copyBackup: error creating latest symlink: %w", err)
-			}
-			s.logger.Infof("Created/Updated symlink `%s` for latest backup.", s.c.BackupLatestSymlink)
-		}
-	}
-	return nil
-}
-
-// pruneOldBackups rotates away backups from local and remote storages using
-// the given configuration. In case the given configuration would delete all
-// backups, it does nothing instead.
-func (s *script) pruneOldBackups() error {
-	if s.c.BackupRetentionDays < 0 {
-		return nil
-	}
-
-	if s.c.BackupPruningLeeway != 0 {
-		s.logger.Infof("Sleeping for %s before pruning backups.", s.c.BackupPruningLeeway)
-		time.Sleep(s.c.BackupPruningLeeway)
-	}
-
-	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays))
-
-	if s.mc != nil {
-		candidates := s.mc.ListObjects(context.Background(), s.c.AwsS3BucketName, minio.ListObjectsOptions{
-			WithMetadata: true,
-			Prefix:       s.c.BackupPruningPrefix,
-		})
-
-		var matches []minio.ObjectInfo
-		var lenCandidates int
-		for candidate := range candidates {
-			lenCandidates++
-			if candidate.Err != nil {
-				return fmt.Errorf(
-					"pruneOldBackups: error looking up candidates from remote storage: %w",
-					candidate.Err,
-				)
-			}
-			if candidate.LastModified.Before(deadline) {
-				matches = append(matches, candidate)
-			}
-		}
-
-		if len(matches) != 0 && len(matches) != lenCandidates {
-			objectsCh := make(chan minio.ObjectInfo)
-			go func() {
-				for _, match := range matches {
-					objectsCh <- match
-				}
-				close(objectsCh)
-			}()
-			errChan := s.mc.RemoveObjects(context.Background(), s.c.AwsS3BucketName, objectsCh, minio.RemoveObjectsOptions{})
-			var removeErrors []error
-			for result := range errChan {
-				if result.Err != nil {
-					removeErrors = append(removeErrors, result.Err)
-				}
-			}
-
-			if len(removeErrors) != 0 {
-				return fmt.Errorf(
-					"pruneOldBackups: %d error(s) removing files from remote storage: %w",
-					len(removeErrors),
-					join(removeErrors...),
-				)
-			}
-			s.logger.Infof(
-				"Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.",
-				len(matches),
-				lenCandidates,
-				s.c.BackupRetentionDays,
-			)
-		} else if len(matches) != 0 && len(matches) == lenCandidates {
-			s.logger.Warnf(
-				"The current configuration would delete all %d remote backup copies.",
-				len(matches),
-			)
-			s.logger.Warn("Refusing to do so, please check your configuration.")
-		} else {
-			s.logger.Infof("None of %d remote backup(s) were pruned.", lenCandidates)
-		}
-	}
-
-	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
-		globPattern := path.Join(
-			s.c.BackupArchive,
-			fmt.Sprintf("%s*", s.c.BackupPruningPrefix),
-		)
-		globMatches, err := filepath.Glob(globPattern)
-		if err != nil {
-			return fmt.Errorf(
-				"pruneOldBackups: error looking up matching files using pattern %s: %w",
-				globPattern,
-				err,
-			)
-		}
-
-		var candidates []string
-		for _, candidate := range globMatches {
-			fi, err := os.Lstat(candidate)
-			if err != nil {
-				return fmt.Errorf(
-					"pruneOldBackups: error calling Lstat on file %s: %w",
-					candidate,
-					err,
-				)
-			}
-
-			if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
-				candidates = append(candidates, candidate)
-			}
-		}
-
-		var matches []string
-		for _, candidate := range candidates {
-			fi, err := os.Stat(candidate)
-			if err != nil {
-				return fmt.Errorf(
-					"pruneOldBackups: error calling stat on file %s: %w",
-					candidate,
-					err,
-				)
-			}
-			if fi.ModTime().Before(deadline) {
-				matches = append(matches, candidate)
-			}
-		}
-
-		if len(matches) != 0 && len(matches) != len(candidates) {
-			var removeErrors []error
-			for _, match := range matches {
-				if err := os.Remove(match); err != nil {
-					removeErrors = append(removeErrors, err)
-				}
-			}
-			if len(removeErrors) != 0 {
-				return fmt.Errorf(
-					"pruneOldBackups: %d error(s) deleting local files, starting with: %w",
-					len(removeErrors),
-					join(removeErrors...),
-				)
-			}
-			s.logger.Infof(
-				"Pruned %d out of %d local backup(s) as their age exceeded the configured retention period of %d days.",
-				len(matches),
-				len(candidates),
-				s.c.BackupRetentionDays,
-			)
-		} else if len(matches) != 0 && len(matches) == len(candidates) {
-			s.logger.Warnf(
-				"The current configuration would delete all %d local backup copies.",
-				len(matches),
-			)
-			s.logger.Warn("Refusing to do so, please check your configuration.")
-		} else {
-			s.logger.Infof("None of %d local backup(s) were pruned.", len(candidates))
-		}
-	}
-	return nil
-}
-
-// runHooks runs all hooks that have been registered using the
-// given levels in the defined ordering. In case executing a hook returns an
-// error, the following hooks will still be run before the function returns.
-func (s *script) runHooks(err error) error {
-	sort.SliceStable(s.hooks, func(i, j int) bool {
-		return s.hooks[i].level < s.hooks[j].level
-	})
-	var actionErrors []error
-	for _, hook := range s.hooks {
-		if hook.level > s.hookLevel {
-			continue
-		}
-		if actionErr := hook.action(err); actionErr != nil {
-			actionErrors = append(actionErrors, fmt.Errorf("runHooks: error running hook: %w", actionErr))
-		}
-	}
-	if len(actionErrors) != 0 {
-		return join(actionErrors...)
-	}
-	return nil
-}
-
-// must exits the script run prematurely in case the given error
-// is non-nil.
-func (s *script) must(err error) {
-	if err != nil {
-		s.logger.Errorf("Fatal error running backup: %s", err)
-		panic(err)
-	}
-}
-
-// remove removes the given file or directory from disk.
-func remove(location string) error {
-	fi, err := os.Lstat(location)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return fmt.Errorf("remove: error checking for existence of `%s`: %w", location, err)
-	}
-	if fi.IsDir() {
-		err = os.RemoveAll(location)
-	} else {
-		err = os.Remove(location)
-	}
-	if err != nil {
-		return fmt.Errorf("remove: error removing `%s`: %w", location, err)
-	}
-	return nil
-}
-
-// lock opens a lockfile at the given location, keeping it locked until the
-// caller invokes the returned release func. When invoked while the file is
-// still locked the function panics.
-func lock(lockfile string) func() error {
-	fileLock := flock.New(lockfile)
-	acquired, err := fileLock.TryLock()
-	if err != nil {
-		panic(err)
-	}
-	if !acquired {
-		panic("unable to acquire file lock")
-	}
-	return fileLock.Unlock
-}
-
-// copy creates a copy of the file located at `dst` at `src`.
-func copyFile(src, dst string) error {
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer in.Close()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-
-	_, err = io.Copy(out, in)
-	if err != nil {
-		out.Close()
-		return err
-	}
-	return out.Close()
-}
-
-// join takes a list of errors and joins them into a single error
-func join(errs ...error) error {
-	if len(errs) == 1 {
-		return errs[0]
-	}
-	var msgs []string
-	for _, err := range errs {
-		if err == nil {
-			continue
-		}
-		msgs = append(msgs, err.Error())
-	}
-	return errors.New("[" + strings.Join(msgs, ", ") + "]")
-}
-
-// buffer takes an io.Writer and returns a wrapped version of the
-// writer that writes to both the original target as well as the returned buffer
-func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {
-	buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w}
-	return buffering, &buffering.buf
-}
-
-type bufferingWriter struct {
-	buf    bytes.Buffer
-	writer io.Writer
-}
-
-func (b *bufferingWriter) Write(p []byte) (n int, err error) {
-	if n, err := b.buf.Write(p); err != nil {
-		return n, fmt.Errorf("bufferingWriter: error writing to buffer: %w", err)
-	}
-	return b.writer.Write(p)
-}
-
-// hook contains a queued action that can be trigger them when the script
-// reaches a certain point (e.g. unsuccessful backup)
-type hook struct {
-	level  hookLevel
-	action func(err error) error
-}
-
-type hookLevel int
-
-const (
-	hookLevelPlumbing hookLevel = iota
-	hookLevelError
-	hookLevelInfo
-)
-
-var hookLevels = map[string]hookLevel{
-	"info":  hookLevelInfo,
-	"error": hookLevelError,
 }

cmd/backup/notifications.go

@@ -0,0 +1,108 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"fmt"
+	"os"
+	"text/template"
+	"time"
+
+	sTypes "github.com/containrrr/shoutrrr/pkg/types"
+)
+
+//go:embed notifications.tmpl
+var defaultNotifications string
+
+// NotificationData data to be passed to the notification templates
+type NotificationData struct {
+	Error  error
+	Config *Config
+	Stats  *Stats
+}
+
+// notify sends a notification using the given title and body templates.
+// Automatically creates notification data, adding the given error
+func (s *script) notify(titleTemplate string, bodyTemplate string, err error) error {
+	params := NotificationData{
+		Error:  err,
+		Stats:  s.stats,
+		Config: s.c,
+	}
+
+	titleBuf := &bytes.Buffer{}
+	if err := s.template.ExecuteTemplate(titleBuf, titleTemplate, params); err != nil {
+		return fmt.Errorf("notify: error executing %s template: %w", titleTemplate, err)
+	}
+
+	bodyBuf := &bytes.Buffer{}
+	if err := s.template.ExecuteTemplate(bodyBuf, bodyTemplate, params); err != nil {
+		return fmt.Errorf("notify: error executing %s template: %w", bodyTemplate, err)
+	}
+
+	if err := s.sendNotification(titleBuf.String(), bodyBuf.String()); err != nil {
+		return fmt.Errorf("notify: error notifying: %w", err)
+	}
+	return nil
+}
+
+// notifyFailure sends a notification about a failed backup run
+func (s *script) notifyFailure(err error) error {
+	return s.notify("title_failure", "body_failure", err)
+}
+
+// notifyFailure sends a notification about a successful backup run
+func (s *script) notifySuccess() error {
+	return s.notify("title_success", "body_success", nil)
+}
+
+// sendNotification sends a notification to all configured third party services
+func (s *script) sendNotification(title, body string) error {
+	var errs []error
+	for _, result := range s.sender.Send(body, &sTypes.Params{"title": title}) {
+		if result != nil {
+			errs = append(errs, result)
+		}
+	}
+	if len(errs) != 0 {
+		return fmt.Errorf("sendNotification: error sending message: %w", errors.Join(errs...))
+	}
+	return nil
+}
+
+var templateHelpers = template.FuncMap{
+	"formatTime": func(t time.Time) string {
+		return t.Format(time.RFC3339)
+	},
+	"formatBytesDec": func(bytes uint64) string {
+		return formatBytes(bytes, true)
+	},
+	"formatBytesBin": func(bytes uint64) string {
+		return formatBytes(bytes, false)
+	},
+	"env": os.Getenv,
+}
+
+// formatBytes converts an amount of bytes in a human-readable representation
+// the decimal parameter specifies if using powers of 1000 (decimal) or powers of 1024 (binary)
+func formatBytes(b uint64, decimal bool) string {
+	unit := uint64(1024)
+	format := "%.1f %ciB"
+	if decimal {
+		unit = uint64(1000)
+		format = "%.1f %cB"
+	}
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := unit, 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf(format, float64(b)/float64(div), "kMGTPE"[exp])
+}

cmd/backup/notifications.tmpl

@@ -0,0 +1,26 @@
+{{ define "title_failure" -}}
+Failure running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
+{{- end }}
+
+
+{{ define "body_failure" -}}
+Running docker-volume-backup failed with error: {{ .Error }}
+
+Log output of the failed run was:
+
+{{ .Stats.LogOutput }}
+{{- end }}
+
+
+{{ define "title_success" -}}
+Success running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
+{{- end }}
+
+
+{{ define "body_success" -}}
+Running docker-volume-backup succeeded.
+
+Log output was:
+
+{{ .Stats.LogOutput }}
+{{- end }}

cmd/backup/script.go

@@ -0,0 +1,657 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path"
+	"path/filepath"
+	"slices"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/offen/docker-volume-backup/internal/storage/azure"
+	"github.com/offen/docker-volume-backup/internal/storage/dropbox"
+	"github.com/offen/docker-volume-backup/internal/storage/local"
+	"github.com/offen/docker-volume-backup/internal/storage/s3"
+	"github.com/offen/docker-volume-backup/internal/storage/ssh"
+	"github.com/offen/docker-volume-backup/internal/storage/webdav"
+
+	"github.com/ProtonMail/go-crypto/openpgp"
+	"github.com/containrrr/shoutrrr"
+	"github.com/containrrr/shoutrrr/pkg/router"
+	"github.com/docker/docker/api/types"
+	ctr "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+	"github.com/leekchan/timeutil"
+	"github.com/offen/envconfig"
+	"github.com/otiai10/copy"
+	"golang.org/x/sync/errgroup"
+)
+
+// script holds all the stateful information required to orchestrate a
+// single backup run.
+type script struct {
+	cli       *client.Client
+	storages  []storage.Backend
+	logger    *slog.Logger
+	sender    *router.ServiceRouter
+	template  *template.Template
+	hooks     []hook
+	hookLevel hookLevel
+
+	file  string
+	stats *Stats
+
+	encounteredLock bool
+
+	c *Config
+}
+
+// newScript creates all resources needed for the script to perform actions against
+// remote resources like the Docker engine or remote storage locations. All
+// reading from env vars or other configuration sources is expected to happen
+// in this method.
+func newScript() (*script, error) {
+	stdOut, logBuffer := buffer(os.Stdout)
+	s := &script{
+		c:      &Config{},
+		logger: slog.New(slog.NewTextHandler(stdOut, nil)),
+		stats: &Stats{
+			StartTime: time.Now(),
+			LogOutput: logBuffer,
+			Storages: map[string]StorageStats{
+				"S3":      {},
+				"WebDAV":  {},
+				"SSH":     {},
+				"Local":   {},
+				"Azure":   {},
+				"Dropbox": {},
+			},
+		},
+	}
+
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		s.stats.EndTime = time.Now()
+		s.stats.TookTime = s.stats.EndTime.Sub(s.stats.StartTime)
+		return nil
+	})
+
+	envconfig.Lookup = func(key string) (string, bool) {
+		value, okValue := os.LookupEnv(key)
+		location, okFile := os.LookupEnv(key + "_FILE")
+
+		switch {
+		case okValue && !okFile: // only value
+			return value, true
+		case !okValue && okFile: // only file
+			contents, err := os.ReadFile(location)
+			if err != nil {
+				s.must(fmt.Errorf("newScript: failed to read %s! Error: %s", location, err))
+				return "", false
+			}
+			return string(contents), true
+		case okValue && okFile: // both
+			s.must(fmt.Errorf("newScript: both %s and %s are set!", key, key+"_FILE"))
+			return "", false
+		default: // neither, ignore
+			return "", false
+		}
+	}
+
+	if err := envconfig.Process("", s.c); err != nil {
+		return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
+	}
+
+	s.file = path.Join("/tmp", s.c.BackupFilename)
+
+	tmplFileName, tErr := template.New("extension").Parse(s.file)
+	if tErr != nil {
+		return nil, fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr)
+	}
+
+	var bf bytes.Buffer
+	if tErr := tmplFileName.Execute(&bf, map[string]string{
+		"Extension": fmt.Sprintf("tar.%s", s.c.BackupCompression),
+	}); tErr != nil {
+		return nil, fmt.Errorf("newScript: error executing backup file extension template: %w", tErr)
+	}
+	s.file = bf.String()
+
+	if s.c.BackupFilenameExpand {
+		s.file = os.ExpandEnv(s.file)
+		s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
+		s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix)
+	}
+	s.file = timeutil.Strftime(&s.stats.StartTime, s.file)
+
+	_, err := os.Stat("/var/run/docker.sock")
+	_, dockerHostSet := os.LookupEnv("DOCKER_HOST")
+	if !os.IsNotExist(err) || dockerHostSet {
+		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+		if err != nil {
+			return nil, fmt.Errorf("newScript: failed to create docker client")
+		}
+		s.cli = cli
+	}
+
+	logFunc := func(logType storage.LogLevel, context string, msg string, params ...any) {
+		switch logType {
+		case storage.LogLevelWarning:
+			s.logger.Warn(fmt.Sprintf(msg, params...), "storage", context)
+		case storage.LogLevelError:
+			s.logger.Error(fmt.Sprintf(msg, params...), "storage", context)
+		default:
+			s.logger.Info(fmt.Sprintf(msg, params...), "storage", context)
+		}
+	}
+
+	if s.c.AwsS3BucketName != "" {
+		s3Config := s3.Config{
+			Endpoint:         s.c.AwsEndpoint,
+			AccessKeyID:      s.c.AwsAccessKeyID,
+			SecretAccessKey:  s.c.AwsSecretAccessKey,
+			IamRoleEndpoint:  s.c.AwsIamRoleEndpoint,
+			EndpointProto:    s.c.AwsEndpointProto,
+			EndpointInsecure: s.c.AwsEndpointInsecure,
+			RemotePath:       s.c.AwsS3Path,
+			BucketName:       s.c.AwsS3BucketName,
+			StorageClass:     s.c.AwsStorageClass,
+			CACert:           s.c.AwsEndpointCACert.Cert,
+			PartSize:         s.c.AwsPartSize,
+		}
+		if s3Backend, err := s3.NewStorageBackend(s3Config, logFunc); err != nil {
+			return nil, fmt.Errorf("newScript: error creating s3 storage backend: %w", err)
+		} else {
+			s.storages = append(s.storages, s3Backend)
+		}
+	}
+
+	if s.c.WebdavUrl != "" {
+		webDavConfig := webdav.Config{
+			URL:         s.c.WebdavUrl,
+			URLInsecure: s.c.WebdavUrlInsecure,
+			Username:    s.c.WebdavUsername,
+			Password:    s.c.WebdavPassword,
+			RemotePath:  s.c.WebdavPath,
+		}
+		if webdavBackend, err := webdav.NewStorageBackend(webDavConfig, logFunc); err != nil {
+			return nil, fmt.Errorf("newScript: error creating webdav storage backend: %w", err)
+		} else {
+			s.storages = append(s.storages, webdavBackend)
+		}
+	}
+
+	if s.c.SSHHostName != "" {
+		sshConfig := ssh.Config{
+			HostName:           s.c.SSHHostName,
+			Port:               s.c.SSHPort,
+			User:               s.c.SSHUser,
+			Password:           s.c.SSHPassword,
+			IdentityFile:       s.c.SSHIdentityFile,
+			IdentityPassphrase: s.c.SSHIdentityPassphrase,
+			RemotePath:         s.c.SSHRemotePath,
+		}
+		if sshBackend, err := ssh.NewStorageBackend(sshConfig, logFunc); err != nil {
+			return nil, fmt.Errorf("newScript: error creating ssh storage backend: %w", err)
+		} else {
+			s.storages = append(s.storages, sshBackend)
+		}
+	}
+
+	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
+		localConfig := local.Config{
+			ArchivePath:   s.c.BackupArchive,
+			LatestSymlink: s.c.BackupLatestSymlink,
+		}
+		localBackend := local.NewStorageBackend(localConfig, logFunc)
+		s.storages = append(s.storages, localBackend)
+	}
+
+	if s.c.AzureStorageAccountName != "" {
+		azureConfig := azure.Config{
+			ContainerName:     s.c.AzureStorageContainerName,
+			AccountName:       s.c.AzureStorageAccountName,
+			PrimaryAccountKey: s.c.AzureStoragePrimaryAccountKey,
+			Endpoint:          s.c.AzureStorageEndpoint,
+			RemotePath:        s.c.AzureStoragePath,
+		}
+		azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: error creating azure storage backend: %w", err)
+		}
+		s.storages = append(s.storages, azureBackend)
+	}
+
+	if s.c.DropboxRefreshToken != "" && s.c.DropboxAppKey != "" && s.c.DropboxAppSecret != "" {
+		dropboxConfig := dropbox.Config{
+			Endpoint:         s.c.DropboxEndpoint,
+			OAuth2Endpoint:   s.c.DropboxOAuth2Endpoint,
+			RefreshToken:     s.c.DropboxRefreshToken,
+			AppKey:           s.c.DropboxAppKey,
+			AppSecret:        s.c.DropboxAppSecret,
+			RemotePath:       s.c.DropboxRemotePath,
+			ConcurrencyLevel: s.c.DropboxConcurrencyLevel.Int(),
+		}
+		dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: error creating dropbox storage backend: %w", err)
+		}
+		s.storages = append(s.storages, dropboxBackend)
+	}
+
+	if s.c.EmailNotificationRecipient != "" {
+		emailURL := fmt.Sprintf(
+			"smtp://%s:%s@%s:%d/?from=%s&to=%s",
+			s.c.EmailSMTPUsername,
+			s.c.EmailSMTPPassword,
+			s.c.EmailSMTPHost,
+			s.c.EmailSMTPPort,
+			s.c.EmailNotificationSender,
+			s.c.EmailNotificationRecipient,
+		)
+		s.c.NotificationURLs = append(s.c.NotificationURLs, emailURL)
+		s.logger.Warn(
+			"Using EMAIL_* keys for providing notification configuration has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use NOTIFICATION_URLS instead. Refer to the README for an upgrade guide.",
+		)
+	}
+
+	hookLevel, ok := hookLevels[s.c.NotificationLevel]
+	if !ok {
+		return nil, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel)
+	}
+	s.hookLevel = hookLevel
+
+	if len(s.c.NotificationURLs) > 0 {
+		sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...)
+		if senderErr != nil {
+			return nil, fmt.Errorf("newScript: error creating sender: %w", senderErr)
+		}
+		s.sender = sender
+
+		tmpl := template.New("")
+		tmpl.Funcs(templateHelpers)
+		tmpl, err = tmpl.Parse(defaultNotifications)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
+		}
+
+		if fi, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil && fi.IsDir() {
+			tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*")
+			if err != nil {
+				return nil, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
+			}
+		}
+		s.template = tmpl
+
+		// To prevent duplicate notifications, ensure the regsistered callbacks
+		// run mutually exclusive.
+		s.registerHook(hookLevelError, func(err error) error {
+			if err == nil {
+				return nil
+			}
+			return s.notifyFailure(err)
+		})
+		s.registerHook(hookLevelInfo, func(err error) error {
+			if err != nil {
+				return nil
+			}
+			return s.notifySuccess()
+		})
+	}
+
+	return s, nil
+}
+
+// stopContainers stops all Docker containers that are marked as to being
+// stopped during the backup and returns a function that can be called to
+// restart everything that has been stopped.
+func (s *script) stopContainers() (func() error, error) {
+	if s.cli == nil {
+		return noop, nil
+	}
+
+	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{})
+	if err != nil {
+		return noop, fmt.Errorf("stopContainers: error querying for containers: %w", err)
+	}
+
+	containerLabel := fmt.Sprintf(
+		"docker-volume-backup.stop-during-backup=%s",
+		s.c.BackupStopContainerLabel,
+	)
+	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Filters: filters.NewArgs(filters.KeyValuePair{
+			Key:   "label",
+			Value: containerLabel,
+		}),
+	})
+
+	if err != nil {
+		return noop, fmt.Errorf("stopContainers: error querying for containers to stop: %w", err)
+	}
+
+	if len(containersToStop) == 0 {
+		return noop, nil
+	}
+
+	s.logger.Info(
+		fmt.Sprintf(
+			"Stopping %d container(s) labeled `%s` out of %d running container(s).",
+			len(containersToStop),
+			containerLabel,
+			len(allContainers),
+		),
+	)
+
+	var stoppedContainers []types.Container
+	var stopErrors []error
+	for _, container := range containersToStop {
+		if err := s.cli.ContainerStop(context.Background(), container.ID, ctr.StopOptions{}); err != nil {
+			stopErrors = append(stopErrors, err)
+		} else {
+			stoppedContainers = append(stoppedContainers, container)
+		}
+	}
+
+	var stopError error
+	if len(stopErrors) != 0 {
+		stopError = fmt.Errorf(
+			"stopContainers: %d error(s) stopping containers: %w",
+			len(stopErrors),
+			errors.Join(stopErrors...),
+		)
+	}
+
+	s.stats.Containers = ContainersStats{
+		All:     uint(len(allContainers)),
+		ToStop:  uint(len(containersToStop)),
+		Stopped: uint(len(stoppedContainers)),
+	}
+
+	return func() error {
+		servicesRequiringUpdate := map[string]struct{}{}
+
+		var restartErrors []error
+		for _, container := range stoppedContainers {
+			if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok {
+				servicesRequiringUpdate[swarmServiceName] = struct{}{}
+				continue
+			}
+			if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil {
+				restartErrors = append(restartErrors, err)
+			}
+		}
+
+		if len(servicesRequiringUpdate) != 0 {
+			services, _ := s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
+			for serviceName := range servicesRequiringUpdate {
+				var serviceMatch swarm.Service
+				for _, service := range services {
+					if service.Spec.Name == serviceName {
+						serviceMatch = service
+						break
+					}
+				}
+				if serviceMatch.ID == "" {
+					return fmt.Errorf("stopContainers: couldn't find service with name %s", serviceName)
+				}
+				serviceMatch.Spec.TaskTemplate.ForceUpdate += 1
+				if _, err := s.cli.ServiceUpdate(
+					context.Background(), serviceMatch.ID,
+					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},
+				); err != nil {
+					restartErrors = append(restartErrors, err)
+				}
+			}
+		}
+
+		if len(restartErrors) != 0 {
+			return fmt.Errorf(
+				"stopContainers: %d error(s) restarting containers and services: %w",
+				len(restartErrors),
+				errors.Join(restartErrors...),
+			)
+		}
+		s.logger.Info(
+			fmt.Sprintf(
+				"Restarted %d container(s) and the matching service(s).",
+				len(stoppedContainers),
+			),
+		)
+		return nil
+	}, stopError
+}
+
+// createArchive creates a tar archive of the configured backup location and
+// saves it to disk.
+func (s *script) createArchive() error {
+	backupSources := s.c.BackupSources
+
+	if s.c.BackupFromSnapshot {
+		s.logger.Warn(
+			"Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the README for an upgrade guide.",
+		)
+		backupSources = filepath.Join("/tmp", s.c.BackupSources)
+		// copy before compressing guard against a situation where backup folder's content are still growing.
+		s.registerHook(hookLevelPlumbing, func(error) error {
+			if err := remove(backupSources); err != nil {
+				return fmt.Errorf("createArchive: error removing snapshot: %w", err)
+			}
+			s.logger.Info(
+				fmt.Sprintf("Removed snapshot `%s`.", backupSources),
+			)
+			return nil
+		})
+		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
+			PreserveTimes: true,
+			PreserveOwner: true,
+		}); err != nil {
+			return fmt.Errorf("createArchive: error creating snapshot: %w", err)
+		}
+		s.logger.Info(
+			fmt.Sprintf("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources),
+		)
+	}
+
+	tarFile := s.file
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		if err := remove(tarFile); err != nil {
+			return fmt.Errorf("createArchive: error removing tar file: %w", err)
+		}
+		s.logger.Info(
+			fmt.Sprintf("Removed tar file `%s`.", tarFile),
+		)
+		return nil
+	})
+
+	backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources))
+	if err != nil {
+		return fmt.Errorf("createArchive: error getting absolute path: %w", err)
+	}
+
+	var filesEligibleForBackup []string
+	if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) {
+			return nil
+		}
+		filesEligibleForBackup = append(filesEligibleForBackup, path)
+		return nil
+	}); err != nil {
+		return fmt.Errorf("createArchive: error walking filesystem tree: %w", err)
+	}
+
+	if err := createArchive(filesEligibleForBackup, backupSources, tarFile, s.c.BackupCompression.String(), s.c.GzipParallelism.Int()); err != nil {
+		return fmt.Errorf("createArchive: error compressing backup folder: %w", err)
+	}
+
+	s.logger.Info(
+		fmt.Sprintf("Created backup of `%s` at `%s`.", backupSources, tarFile),
+	)
+	return nil
+}
+
+// encryptArchive encrypts the backup file using PGP and the configured passphrase.
+// In case no passphrase is given it returns early, leaving the backup file
+// untouched.
+func (s *script) encryptArchive() error {
+	if s.c.GpgPassphrase == "" {
+		return nil
+	}
+
+	gpgFile := fmt.Sprintf("%s.gpg", s.file)
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		if err := remove(gpgFile); err != nil {
+			return fmt.Errorf("encryptArchive: error removing gpg file: %w", err)
+		}
+		s.logger.Info(
+			fmt.Sprintf("Removed GPG file `%s`.", gpgFile),
+		)
+		return nil
+	})
+
+	outFile, err := os.Create(gpgFile)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error opening out file: %w", err)
+	}
+	defer outFile.Close()
+
+	_, name := path.Split(s.file)
+	dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
+		IsBinary: true,
+		FileName: name,
+	}, nil)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error encrypting backup file: %w", err)
+	}
+	defer dst.Close()
+
+	src, err := os.Open(s.file)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error opening backup file `%s`: %w", s.file, err)
+	}
+
+	if _, err := io.Copy(dst, src); err != nil {
+		return fmt.Errorf("encryptArchive: error writing ciphertext to file: %w", err)
+	}
+
+	s.file = gpgFile
+	s.logger.Info(
+		fmt.Sprintf("Encrypted backup using given passphrase, saving as `%s`.", s.file),
+	)
+	return nil
+}
+
+// copyArchive makes sure the backup file is copied to both local and remote locations
+// as per the given configuration.
+func (s *script) copyArchive() error {
+	_, name := path.Split(s.file)
+	if stat, err := os.Stat(s.file); err != nil {
+		return fmt.Errorf("copyArchive: unable to stat backup file: %w", err)
+	} else {
+		size := stat.Size()
+		s.stats.BackupFile = BackupFileStats{
+			Size:     uint64(size),
+			Name:     name,
+			FullPath: s.file,
+		}
+	}
+
+	eg := errgroup.Group{}
+	for _, backend := range s.storages {
+		b := backend
+		eg.Go(func() error {
+			return b.Copy(s.file)
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return fmt.Errorf("copyArchive: error copying archive: %w", err)
+	}
+
+	return nil
+}
+
+// pruneBackups rotates away backups from local and remote storages using
+// the given configuration. In case the given configuration would delete all
+// backups, it does nothing instead and logs a warning.
+func (s *script) pruneBackups() error {
+	if s.c.BackupRetentionDays < 0 {
+		return nil
+	}
+
+	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
+
+	eg := errgroup.Group{}
+	for _, backend := range s.storages {
+		b := backend
+		eg.Go(func() error {
+			if skipPrune(b.Name(), s.c.BackupSkipBackendsFromPrune) {
+				s.logger.Info(
+					fmt.Sprintf("Skipping pruning for backend `%s`.", b.Name()),
+				)
+				return nil
+			}
+			stats, err := b.Prune(deadline, s.c.BackupPruningPrefix)
+			if err != nil {
+				return err
+			}
+			s.stats.Lock()
+			s.stats.Storages[b.Name()] = StorageStats{
+				Total:  stats.Total,
+				Pruned: stats.Pruned,
+			}
+			s.stats.Unlock()
+			return nil
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return fmt.Errorf("pruneBackups: error pruning backups: %w", err)
+	}
+
+	return nil
+}
+
+// must exits the script run prematurely in case the given error
+// is non-nil.
+func (s *script) must(err error) {
+	if err != nil {
+		s.logger.Error(
+			fmt.Sprintf("Fatal error running backup: %s", err),
+		)
+		panic(err)
+	}
+}
+
+// skipPrune returns true if the given backend name is contained in the
+// list of skipped backends.
+func skipPrune(name string, skippedBackends []string) bool {
+	return slices.ContainsFunc(
+		skippedBackends,
+		func(b string) bool {
+			return strings.EqualFold(b, name) // ignore case on both sides
+		},
+	)
+}

cmd/backup/stats.go

@@ -0,0 +1,45 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	"sync"
+	"time"
+)
+
+// ContainersStats stats about the docker containers
+type ContainersStats struct {
+	All        uint
+	ToStop     uint
+	Stopped    uint
+	StopErrors uint
+}
+
+// BackupFileStats stats about the created backup file
+type BackupFileStats struct {
+	Name     string
+	FullPath string
+	Size     uint64
+}
+
+// StorageStats stats about the status of an archival directory
+type StorageStats struct {
+	Total       uint
+	Pruned      uint
+	PruneErrors uint
+}
+
+// Stats global stats regarding script execution
+type Stats struct {
+	sync.Mutex
+	StartTime  time.Time
+	EndTime    time.Time
+	TookTime   time.Duration
+	LockedTime time.Duration
+	LogOutput  *bytes.Buffer
+	Containers ContainersStats
+	BackupFile BackupFileStats
+	Storages   map[string]StorageStats
+}

cmd/backup/util.go

@@ -0,0 +1,52 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+)
+
+var noop = func() error { return nil }
+
+// remove removes the given file or directory from disk.
+func remove(location string) error {
+	fi, err := os.Lstat(location)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("remove: error checking for existence of `%s`: %w", location, err)
+	}
+	if fi.IsDir() {
+		err = os.RemoveAll(location)
+	} else {
+		err = os.Remove(location)
+	}
+	if err != nil {
+		return fmt.Errorf("remove: error removing `%s`: %w", location, err)
+	}
+	return nil
+}
+
+// buffer takes an io.Writer and returns a wrapped version of the
+// writer that writes to both the original target as well as the returned buffer
+func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {
+	buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w}
+	return buffering, &buffering.buf
+}
+
+type bufferingWriter struct {
+	buf    bytes.Buffer
+	writer io.Writer
+}
+
+func (b *bufferingWriter) Write(p []byte) (n int, err error) {
+	if n, err := b.buf.Write(p); err != nil {
+		return n, fmt.Errorf("(*bufferingWriter).Write: error writing to buffer: %w", err)
+	}
+	return b.writer.Write(p)
+}

docs/NOTIFICATION-TEMPLATES.md

@@ -0,0 +1,40 @@
+# Notification templates reference
+
+In order to customize title and body of notifications you'll have to write a [go template](https://pkg.go.dev/text/template) and mount it inside the `/etc/dockervolumebackup/notifications.d/` directory.
+
+Configuration, data about the backup run and helper functions will be passed to this template, this page documents them fully.
+
+## Data
+Here is a list of all data passed to the template:
+
+* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`)
+* `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates
+* `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available.
+  * `StartTime`: time when the script started execution
+  * `EndTime`: time when the backup has completed successfully (after pruning)
+  * `TookTime`: amount of time it took for the backup to run. (equal to `EndTime - StartTime`)
+  * `LockedTime`: amount of time it took for the backup to acquire the exclusive lock
+  * `LogOutput`: full log of the application
+  * `Containers`: object containing stats about the docker containers
+    * `All`: total number of containers
+    * `ToStop`: number of containers matched by the stop rule
+    * `Stopped`: number of containers successfully stopped
+    * `StopErrors`: number of containers that were unable to be stopped (equal to `ToStop - Stopped`)
+  * `BackupFile`: object containing information about the backup file
+    * `Name`: name of the backup file (e.g. `backup-2022-02-11T01-00-00.tar.gz`)
+    * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
+    * `Size`: size in bytes of the backup file
+  * `Storages`: object that holds stats about each storage
+    * `Local`, `S3`, `WebDAV`, `Azure`, `Dropbox` or `SSH`:
+      * `Total`: total number of backup files
+      * `Pruned`: number of backup files that were deleted due to pruning rule
+      * `PruneErrors`: number of backup files that were unable to be pruned
+
+## Functions
+
+Some formatting and helper functions are also available:
+
+* `formatTime`: formats a time object using [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format (e.g. `2022-02-11T01:00:00Z`)
+* `formatBytesBin`: formats an amount of bytes using powers of 1024 (e.g. `7055258` bytes will be `6.7 MiB`) 
+* `formatBytesDec`: formats an amount of bytes using powers of 1000 (e.g. `7055258` bytes will be `7.1 MB`)
+* `env`: returns the value of the environment variable of the given key if set

entrypoint.sh

@@ -5,10 +5,22 @@
 
 set -e
 
+if [ ! -d "/etc/dockervolumebackup/conf.d" ]; then
   BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
 
   echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
   echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
+else
+  echo "/etc/dockervolumebackup/conf.d was found, using configuration files from this directory."
+
+  crontab -r && crontab /dev/null
+  for file in /etc/dockervolumebackup/conf.d/*; do
+    source $file
+    BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+    echo "Appending cron.d entry with expression $BACKUP_CRON_EXPRESSION and configuration file $file"
+    (crontab -l; echo "$BACKUP_CRON_EXPRESSION /bin/sh -c 'set -a; source $file; set +a && backup' 2>&1") | crontab -
+  done
+fi
 
 echo "Starting cron in foreground."
-crond -f -l 8
+crond -f -d 8

go.mod

@@ -1,58 +1,72 @@
 module github.com/offen/docker-volume-backup
 
-go 1.17
+go 1.21
 
 require (
-	github.com/containrrr/shoutrrr v0.5.2
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
-	github.com/docker/docker v20.10.11+incompatible
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
+	github.com/containrrr/shoutrrr v0.7.1
+	github.com/cosiner/argv v0.1.0
+	github.com/docker/docker v24.0.5+incompatible
 	github.com/gofrs/flock v0.8.1
-	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/klauspost/compress v1.16.7
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
-	github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615
+	github.com/minio/minio-go/v7 v7.0.62
-	github.com/minio/minio-go/v7 v7.0.16
+	github.com/offen/envconfig v1.5.0
-	github.com/otiai10/copy v1.7.0
+	github.com/otiai10/copy v1.11.0
-	github.com/sirupsen/logrus v1.8.1
+	github.com/pkg/sftp v1.13.6
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
+	github.com/studio-b12/gowebdav v0.9.0
+	golang.org/x/crypto v0.12.0
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783
+	golang.org/x/sync v0.3.0
 )
 
 require (
-	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
-	github.com/containerd/containerd v1.5.5 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/docker/distribution v2.7.1+incompatible // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+)
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95
+	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
-	github.com/fatih/color v1.10.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fatih/color v1.13.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/klauspost/pgzip v1.2.6
-	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/kr/fs v0.1.0 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/term v0.0.0-20200312100748-672ec06f55cd // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/nxadm/tail v1.4.6 // indirect
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	github.com/onsi/ginkgo v1.14.2 // indirect
-	github.com/onsi/gomega v1.10.3 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rs/xid v1.3.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
-	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
-	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
+	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/text v0.12.0 // indirect
-	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
-	google.golang.org/grpc v1.33.2 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	gotest.tools/v3 v3.0.3 // indirect
-	gopkg.in/ini.v1 v1.65.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

internal/storage/azure/azure.go

@@ -0,0 +1,160 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package azure
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"text/template"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/offen/docker-volume-backup/internal/storage"
+)
+
+type azureBlobStorage struct {
+	*storage.StorageBackend
+	client        *azblob.Client
+	containerName string
+}
+
+// Config contains values that define the configuration of an Azure Blob Storage.
+type Config struct {
+	AccountName       string
+	ContainerName     string
+	PrimaryAccountKey string
+	Endpoint          string
+	RemotePath        string
+}
+
+// NewStorageBackend creates and initializes a new Azure Blob Storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error parsing endpoint template: %w", err)
+	}
+	var ep bytes.Buffer
+	if err := endpointTemplate.Execute(&ep, opts); err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err)
+	}
+	normalizedEndpoint := fmt.Sprintf("%s/", strings.TrimSuffix(ep.String(), "/"))
+
+	var client *azblob.Client
+	if opts.PrimaryAccountKey != "" {
+		cred, err := azblob.NewSharedKeyCredential(opts.AccountName, opts.PrimaryAccountKey)
+		if err != nil {
+			return nil, fmt.Errorf("NewStorageBackend: error creating shared key Azure credential: %w", err)
+		}
+
+		client, err = azblob.NewClientWithSharedKeyCredential(normalizedEndpoint, cred, nil)
+		if err != nil {
+			return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err)
+		}
+	} else {
+		cred, err := azidentity.NewManagedIdentityCredential(nil)
+		if err != nil {
+			return nil, fmt.Errorf("NewStorageBackend: error creating managed identity credential: %w", err)
+		}
+		client, err = azblob.NewClient(normalizedEndpoint, cred, nil)
+		if err != nil {
+			return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err)
+		}
+	}
+
+	storage := azureBlobStorage{
+		client:        client,
+		containerName: opts.ContainerName,
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.RemotePath,
+			Log:             logFunc,
+		},
+	}
+	return &storage, nil
+}
+
+// Name returns the name of the storage backend
+func (b *azureBlobStorage) Name() string {
+	return "Azure"
+}
+
+// Copy copies the given file to the storage backend.
+func (b *azureBlobStorage) Copy(file string) error {
+	fileReader, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err)
+	}
+	_, err = b.client.UploadStream(
+		context.Background(),
+		b.containerName,
+		filepath.Join(b.DestinationPath, filepath.Base(file)),
+		fileReader,
+		nil,
+	)
+	if err != nil {
+		return fmt.Errorf("(*azureBlobStorage).Copy: error uploading file %s: %w", file, err)
+	}
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided
+// deadline for the Azure Blob storage backend.
+func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	lookupPrefix := filepath.Join(b.DestinationPath, pruningPrefix)
+	pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{
+		Prefix: &lookupPrefix,
+	})
+	var matches []string
+	var totalCount uint
+	for pager.More() {
+		resp, err := pager.NextPage(context.Background())
+		if err != nil {
+			return nil, fmt.Errorf("(*azureBlobStorage).Prune: error paging over blobs: %w", err)
+		}
+		for _, v := range resp.Segment.BlobItems {
+			totalCount++
+			if v.Properties.LastModified.Before(deadline) {
+				matches = append(matches, *v.Name)
+			}
+		}
+	}
+
+	stats := storage.PruneStats{
+		Total:  totalCount,
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), int(totalCount), func() error {
+		wg := sync.WaitGroup{}
+		wg.Add(len(matches))
+		var errs []error
+
+		for _, match := range matches {
+			name := match
+			go func() {
+				_, err := b.client.DeleteBlob(context.Background(), b.containerName, name, nil)
+				if err != nil {
+					errs = append(errs, err)
+				}
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+		if len(errs) != 0 {
+			return errors.Join(errs...)
+		}
+		return nil
+	}); err != nil {
+		return &stats, err
+	}
+
+	return &stats, nil
+}

internal/storage/dropbox/dropbox.go

@@ -0,0 +1,260 @@
+package dropbox
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"golang.org/x/oauth2"
+)
+
+type dropboxStorage struct {
+	*storage.StorageBackend
+	client           files.Client
+	concurrencyLevel int
+}
+
+// Config allows to configure a Dropbox storage backend.
+type Config struct {
+	Endpoint         string
+	OAuth2Endpoint   string
+	RefreshToken     string
+	AppKey           string
+	AppSecret        string
+	RemotePath       string
+	ConcurrencyLevel int
+}
+
+// NewStorageBackend creates and initializes a new Dropbox storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	tokenUrl, _ := url.JoinPath(opts.OAuth2Endpoint, "oauth2/token")
+
+	conf := &oauth2.Config{
+		ClientID:     opts.AppKey,
+		ClientSecret: opts.AppSecret,
+		Endpoint: oauth2.Endpoint{
+			TokenURL: tokenUrl,
+		},
+	}
+
+	logFunc(storage.LogLevelInfo, "Dropbox", "Fetching fresh access token for Dropbox storage backend.")
+	tkSource := conf.TokenSource(context.Background(), &oauth2.Token{RefreshToken: opts.RefreshToken})
+	token, err := tkSource.Token()
+	if err != nil {
+		return nil, fmt.Errorf("(*dropboxStorage).NewStorageBackend: Error refreshing token: %w", err)
+	}
+
+	dbxConfig := dropbox.Config{
+		Token: token.AccessToken,
+	}
+
+	if opts.Endpoint != "https://api.dropbox.com/" {
+		dbxConfig.URLGenerator = func(hostType string, namespace string, route string) string {
+			return fmt.Sprintf("%s/%d/%s/%s", opts.Endpoint, 2, namespace, route)
+		}
+	}
+
+	client := files.New(dbxConfig)
+
+	if opts.ConcurrencyLevel < 1 {
+		logFunc(storage.LogLevelWarning, "Dropbox", "Concurrency level must be at least 1! Using 1 instead of %d.", opts.ConcurrencyLevel)
+		opts.ConcurrencyLevel = 1
+	}
+
+	return &dropboxStorage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.RemotePath,
+			Log:             logFunc,
+		},
+		client:           client,
+		concurrencyLevel: opts.ConcurrencyLevel,
+	}, nil
+}
+
+// Name returns the name of the storage backend
+func (b *dropboxStorage) Name() string {
+	return "Dropbox"
+}
+
+// Copy copies the given file to the WebDav storage backend.
+func (b *dropboxStorage) Copy(file string) error {
+	_, name := path.Split(file)
+
+	folderArg := files.NewCreateFolderArg(b.DestinationPath)
+	if _, err := b.client.CreateFolderV2(folderArg); err != nil {
+		switch err := err.(type) {
+		case files.CreateFolderV2APIError:
+			if err.EndpointError.Path.Tag != files.WriteErrorConflict {
+				return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s': %w", b.DestinationPath, err)
+			}
+			b.Log(storage.LogLevelInfo, b.Name(), "Destination path '%s' already exists, no new directory required.", b.DestinationPath)
+		default:
+			return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s': %w", b.DestinationPath, err)
+		}
+	}
+
+	r, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("(*dropboxStorage).Copy: Error opening the file to be uploaded: %w", err)
+	}
+	defer r.Close()
+
+	// Start new upload session and get session id
+
+	b.Log(storage.LogLevelInfo, b.Name(), "Starting upload session for backup '%s' at path '%s'.", file, b.DestinationPath)
+
+	var sessionId string
+	uploadSessionStartArg := files.NewUploadSessionStartArg()
+	uploadSessionStartArg.SessionType = &files.UploadSessionType{Tagged: dropbox.Tagged{Tag: files.UploadSessionTypeConcurrent}}
+	if res, err := b.client.UploadSessionStart(uploadSessionStartArg, nil); err != nil {
+		return fmt.Errorf("(*dropboxStorage).Copy: Error starting the upload session: %w", err)
+	} else {
+		sessionId = res.SessionId
+	}
+
+	// Send the file in 148MB chunks (Dropbox API limit is 150MB, concurrent upload requires a multiple of 4MB though)
+	// Last append can be any size <= 150MB with Close=True
+
+	const chunkSize = 148 * 1024 * 1024 // 148MB
+	var offset uint64 = 0
+	var guard = make(chan struct{}, b.concurrencyLevel)
+	var errorChn = make(chan error, b.concurrencyLevel)
+	var EOFChn = make(chan bool, b.concurrencyLevel)
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+
+loop:
+	for {
+		guard <- struct{}{} // limit concurrency
+		select {
+		case err := <-errorChn: // error from goroutine
+			return err
+		case <-EOFChn: // EOF from goroutine
+			wg.Wait() // wait for all goroutines to finish
+			break loop
+		default:
+		}
+
+		go func() {
+			defer func() {
+				wg.Done()
+				<-guard
+			}()
+			wg.Add(1)
+			chunk := make([]byte, chunkSize)
+
+			mu.Lock() // to preserve offset of chunks
+
+			select {
+			case <-EOFChn:
+				EOFChn <- true // put it back for outer loop
+				mu.Unlock()
+				return // already EOF
+			default:
+			}
+
+			bytesRead, err := r.Read(chunk)
+			if err != nil {
+				errorChn <- fmt.Errorf("(*dropboxStorage).Copy: Error reading the file to be uploaded: %w", err)
+				mu.Unlock()
+				return
+			}
+			chunk = chunk[:bytesRead]
+
+			uploadSessionAppendArg := files.NewUploadSessionAppendArg(
+				files.NewUploadSessionCursor(sessionId, offset),
+			)
+			isEOF := bytesRead < chunkSize
+			uploadSessionAppendArg.Close = isEOF
+			if isEOF {
+				EOFChn <- true
+			}
+			offset += uint64(bytesRead)
+
+			mu.Unlock()
+
+			if err := b.client.UploadSessionAppendV2(uploadSessionAppendArg, bytes.NewReader(chunk)); err != nil {
+				errorChn <- fmt.Errorf("(*dropboxStorage).Copy: Error appending the file to the upload session: %w", err)
+				return
+			}
+		}()
+	}
+
+	// Finish the upload session, commit the file (no new data added)
+
+	_, err = b.client.UploadSessionFinish(
+		files.NewUploadSessionFinishArg(
+			files.NewUploadSessionCursor(sessionId, 0),
+			files.NewCommitInfo(filepath.Join(b.DestinationPath, name)),
+		), nil)
+	if err != nil {
+		return fmt.Errorf("(*dropboxStorage).Copy: Error finishing the upload session: %w", err)
+	}
+
+	b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup '%s' at path '%s'.", file, b.DestinationPath)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the Dropbox storage backend.
+func (b *dropboxStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	var entries []files.IsMetadata
+	res, err := b.client.ListFolder(files.NewListFolderArg(b.DestinationPath))
+	if err != nil {
+		return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err)
+	}
+	entries = append(entries, res.Entries...)
+
+	for res.HasMore {
+		res, err = b.client.ListFolderContinue(files.NewListFolderContinueArg(res.Cursor))
+		if err != nil {
+			return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err)
+		}
+		entries = append(entries, res.Entries...)
+	}
+
+	var matches []*files.FileMetadata
+	var lenCandidates int
+	for _, candidate := range entries {
+		switch candidate := candidate.(type) {
+		case *files.FileMetadata:
+			if !strings.HasPrefix(candidate.Name, pruningPrefix) {
+				continue
+			}
+			lenCandidates++
+			if candidate.ServerModified.Before(deadline) {
+				matches = append(matches, candidate)
+			}
+		default:
+			continue
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(lenCandidates),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), lenCandidates, func() error {
+		for _, match := range matches {
+			if _, err := b.client.DeleteV2(files.NewDeleteArg(filepath.Join(b.DestinationPath, match.Name))); err != nil {
+				return fmt.Errorf("(*dropboxStorage).Prune: Error removing file from Dropbox storage: %w", err)
+			}
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}

internal/storage/local/local.go

@@ -0,0 +1,160 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package local
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+)
+
+type localStorage struct {
+	*storage.StorageBackend
+	latestSymlink string
+}
+
+// Config allows configuration of a local storage backend.
+type Config struct {
+	ArchivePath   string
+	LatestSymlink string
+}
+
+// NewStorageBackend creates and initializes a new local storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) storage.Backend {
+	return &localStorage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.ArchivePath,
+			Log:             logFunc,
+		},
+		latestSymlink: opts.LatestSymlink,
+	}
+}
+
+// Name return the name of the storage backend
+func (b *localStorage) Name() string {
+	return "Local"
+}
+
+// Copy copies the given file to the local storage backend.
+func (b *localStorage) Copy(file string) error {
+	_, name := path.Split(file)
+
+	if err := copyFile(file, path.Join(b.DestinationPath, name)); err != nil {
+		return fmt.Errorf("(*localStorage).Copy: Error copying file to archive: %w", err)
+	}
+	b.Log(storage.LogLevelInfo, b.Name(), "Stored copy of backup `%s` in `%s`.", file, b.DestinationPath)
+
+	if b.latestSymlink != "" {
+		symlink := path.Join(b.DestinationPath, b.latestSymlink)
+		if _, err := os.Lstat(symlink); err == nil {
+			os.Remove(symlink)
+		}
+		if err := os.Symlink(name, symlink); err != nil {
+			return fmt.Errorf("(*localStorage).Copy: error creating latest symlink: %w", err)
+		}
+		b.Log(storage.LogLevelInfo, b.Name(), "Created/Updated symlink `%s` for latest backup.", b.latestSymlink)
+	}
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the local storage backend.
+func (b *localStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	globPattern := path.Join(
+		b.DestinationPath,
+		fmt.Sprintf("%s*", pruningPrefix),
+	)
+	globMatches, err := filepath.Glob(globPattern)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"(*localStorage).Prune: Error looking up matching files using pattern %s: %w",
+			globPattern,
+			err,
+		)
+	}
+
+	var candidates []string
+	for _, candidate := range globMatches {
+		fi, err := os.Lstat(candidate)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"(*localStorage).Prune: Error calling Lstat on file %s: %w",
+				candidate,
+				err,
+			)
+		}
+
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			candidates = append(candidates, candidate)
+		}
+	}
+
+	var matches []string
+	for _, candidate := range candidates {
+		fi, err := os.Stat(candidate)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"(*localStorage).Prune: Error calling stat on file %s: %w",
+				candidate,
+				err,
+			)
+		}
+		if fi.ModTime().Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(len(candidates)),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), len(candidates), func() error {
+		var removeErrors []error
+		for _, match := range matches {
+			if err := os.Remove(match); err != nil {
+				removeErrors = append(removeErrors, err)
+			}
+		}
+		if len(removeErrors) != 0 {
+			return fmt.Errorf(
+				"(*localStorage).Prune: %d error(s) deleting files, starting with: %w",
+				len(removeErrors),
+				errors.Join(removeErrors...),
+			)
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}
+
+// copy creates a copy of the file located at `dst` at `src`.
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}

internal/storage/s3/s3.go

@@ -0,0 +1,194 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package s3
+
+import (
+	"context"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/offen/docker-volume-backup/internal/storage"
+)
+
+type s3Storage struct {
+	*storage.StorageBackend
+	client       *minio.Client
+	bucket       string
+	storageClass string
+	partSize     int64
+}
+
+// Config contains values that define the configuration of a S3 backend.
+type Config struct {
+	Endpoint         string
+	AccessKeyID      string
+	SecretAccessKey  string
+	IamRoleEndpoint  string
+	EndpointProto    string
+	EndpointInsecure bool
+	RemotePath       string
+	BucketName       string
+	StorageClass     string
+	PartSize         int64
+	CACert           *x509.Certificate
+}
+
+// NewStorageBackend creates and initializes a new S3/Minio storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	var creds *credentials.Credentials
+	if opts.AccessKeyID != "" && opts.SecretAccessKey != "" {
+		creds = credentials.NewStaticV4(
+			opts.AccessKeyID,
+			opts.SecretAccessKey,
+			"",
+		)
+	} else if opts.IamRoleEndpoint != "" {
+		creds = credentials.NewIAM(opts.IamRoleEndpoint)
+	} else {
+		return nil, errors.New("NewStorageBackend: AWS_S3_BUCKET_NAME is defined, but no credentials were provided")
+	}
+
+	options := minio.Options{
+		Creds:  creds,
+		Secure: opts.EndpointProto == "https",
+	}
+
+	transport, err := minio.DefaultTransport(true)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: failed to create default minio transport: %w", err)
+	}
+
+	if opts.EndpointInsecure {
+		if !options.Secure {
+			return nil, errors.New("NewStorageBackend: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
+		}
+		transport.TLSClientConfig.InsecureSkipVerify = true
+	} else if opts.CACert != nil {
+		if transport.TLSClientConfig.RootCAs == nil {
+			transport.TLSClientConfig.RootCAs = x509.NewCertPool()
+		}
+		transport.TLSClientConfig.RootCAs.AddCert(opts.CACert)
+	}
+	options.Transport = transport
+
+	mc, err := minio.New(opts.Endpoint, &options)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error setting up minio client: %w", err)
+	}
+
+	return &s3Storage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.RemotePath,
+			Log:             logFunc,
+		},
+		client:       mc,
+		bucket:       opts.BucketName,
+		storageClass: opts.StorageClass,
+		partSize:     opts.PartSize,
+	}, nil
+}
+
+// Name returns the name of the storage backend
+func (v *s3Storage) Name() string {
+	return "S3"
+}
+
+// Copy copies the given file to the S3/Minio storage backend.
+func (b *s3Storage) Copy(file string) error {
+	_, name := path.Split(file)
+	putObjectOptions := minio.PutObjectOptions{
+		ContentType:  "application/tar+gzip",
+		StorageClass: b.storageClass,
+	}
+
+	if b.partSize > 0 {
+		srcFileInfo, err := os.Stat(file)
+		if err != nil {
+			return fmt.Errorf("(*s3Storage).Copy: error reading the local file: %w", err)
+		}
+
+		_, partSize, _, err := minio.OptimalPartInfo(srcFileInfo.Size(), uint64(b.partSize*1024*1024))
+		if err != nil {
+			return fmt.Errorf("(*s3Storage).Copy: error computing the optimal s3 part size: %w", err)
+		}
+
+		putObjectOptions.PartSize = uint64(partSize)
+	}
+
+	if _, err := b.client.FPutObject(context.Background(), b.bucket, filepath.Join(b.DestinationPath, name), file, putObjectOptions); err != nil {
+		if errResp := minio.ToErrorResponse(err); errResp.Message != "" {
+			return fmt.Errorf(
+				"(*s3Storage).Copy: error uploading backup to remote storage: [Message]: '%s', [Code]: %s, [StatusCode]: %d",
+				errResp.Message,
+				errResp.Code,
+				errResp.StatusCode,
+			)
+		}
+		return fmt.Errorf("(*s3Storage).Copy: error uploading backup to remote storage: %w", err)
+	}
+
+	b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to bucket `%s`.", file, b.bucket)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend.
+func (b *s3Storage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates := b.client.ListObjects(context.Background(), b.bucket, minio.ListObjectsOptions{
+		Prefix:    filepath.Join(b.DestinationPath, pruningPrefix),
+		Recursive: true,
+	})
+
+	var matches []minio.ObjectInfo
+	var lenCandidates int
+	for candidate := range candidates {
+		lenCandidates++
+		if candidate.Err != nil {
+			return nil, fmt.Errorf(
+				"(*s3Storage).Prune: error looking up candidates from remote storage! %w",
+				candidate.Err,
+			)
+		}
+		if candidate.LastModified.Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(lenCandidates),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), lenCandidates, func() error {
+		objectsCh := make(chan minio.ObjectInfo)
+		go func() {
+			for _, match := range matches {
+				objectsCh <- match
+			}
+			close(objectsCh)
+		}()
+		errChan := b.client.RemoveObjects(context.Background(), b.bucket, objectsCh, minio.RemoveObjectsOptions{})
+		var removeErrors []error
+		for result := range errChan {
+			if result.Err != nil {
+				removeErrors = append(removeErrors, result.Err)
+			}
+		}
+		if len(removeErrors) != 0 {
+			return errors.Join(removeErrors...)
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}

internal/storage/ssh/ssh.go

@@ -0,0 +1,189 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package ssh
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+type sshStorage struct {
+	*storage.StorageBackend
+	client     *ssh.Client
+	sftpClient *sftp.Client
+	hostName   string
+}
+
+// Config allows to configure a SSH backend.
+type Config struct {
+	HostName           string
+	Port               string
+	User               string
+	Password           string
+	IdentityFile       string
+	IdentityPassphrase string
+	RemotePath         string
+}
+
+// NewStorageBackend creates and initializes a new SSH storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	var authMethods []ssh.AuthMethod
+
+	if opts.Password != "" {
+		authMethods = append(authMethods, ssh.Password(opts.Password))
+	}
+
+	if _, err := os.Stat(opts.IdentityFile); err == nil {
+		key, err := os.ReadFile(opts.IdentityFile)
+		if err != nil {
+			return nil, errors.New("NewStorageBackend: error reading the private key")
+		}
+
+		var signer ssh.Signer
+		if opts.IdentityPassphrase != "" {
+			signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(opts.IdentityPassphrase))
+			if err != nil {
+				return nil, errors.New("NewStorageBackend: error parsing the encrypted private key")
+			}
+			authMethods = append(authMethods, ssh.PublicKeys(signer))
+		} else {
+			signer, err = ssh.ParsePrivateKey(key)
+			if err != nil {
+				return nil, errors.New("NewStorageBackend: error parsing the private key")
+			}
+			authMethods = append(authMethods, ssh.PublicKeys(signer))
+		}
+	}
+
+	sshClientConfig := &ssh.ClientConfig{
+		User:            opts.User,
+		Auth:            authMethods,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	sshClient, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", opts.HostName, opts.Port), sshClientConfig)
+
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error creating ssh client: %w", err)
+	}
+	_, _, err = sshClient.SendRequest("keepalive", false, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	sftpClient, err := sftp.NewClient(sshClient)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error creating sftp client: %w", err)
+	}
+
+	return &sshStorage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.RemotePath,
+			Log:             logFunc,
+		},
+		client:     sshClient,
+		sftpClient: sftpClient,
+		hostName:   opts.HostName,
+	}, nil
+}
+
+// Name returns the name of the storage backend
+func (b *sshStorage) Name() string {
+	return "SSH"
+}
+
+// Copy copies the given file to the SSH storage backend.
+func (b *sshStorage) Copy(file string) error {
+	source, err := os.Open(file)
+	_, name := path.Split(file)
+	if err != nil {
+		return fmt.Errorf("(*sshStorage).Copy: error reading the file to be uploaded: %w", err)
+	}
+	defer source.Close()
+
+	destination, err := b.sftpClient.Create(filepath.Join(b.DestinationPath, name))
+	if err != nil {
+		return fmt.Errorf("(*sshStorage).Copy: error creating file: %w", err)
+	}
+	defer destination.Close()
+
+	chunk := make([]byte, 1000000)
+	for {
+		num, err := source.Read(chunk)
+		if err == io.EOF {
+			tot, err := destination.Write(chunk[:num])
+			if err != nil {
+				return fmt.Errorf("(*sshStorage).Copy: error uploading the file: %w", err)
+			}
+
+			if tot != len(chunk[:num]) {
+				return errors.New("(*sshStorage).Copy: failed to write stream")
+			}
+
+			break
+		}
+
+		if err != nil {
+			return fmt.Errorf("(*sshStorage).Copy: error uploading the file: %w", err)
+		}
+
+		tot, err := destination.Write(chunk[:num])
+		if err != nil {
+			return fmt.Errorf("(*sshStorage).Copy: error uploading the file: %w", err)
+		}
+
+		if tot != len(chunk[:num]) {
+			return fmt.Errorf("(*sshStorage).Copy: failed to write stream")
+		}
+	}
+
+	b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to '%s' at path '%s'.", file, b.hostName, b.DestinationPath)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the SSH storage backend.
+func (b *sshStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates, err := b.sftpClient.ReadDir(b.DestinationPath)
+	if err != nil {
+		return nil, fmt.Errorf("(*sshStorage).Prune: error reading directory: %w", err)
+	}
+
+	var matches []string
+	for _, candidate := range candidates {
+		if !strings.HasPrefix(candidate.Name(), pruningPrefix) {
+			continue
+		}
+		if candidate.ModTime().Before(deadline) {
+			matches = append(matches, candidate.Name())
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(len(candidates)),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), len(candidates), func() error {
+		for _, match := range matches {
+			if err := b.sftpClient.Remove(filepath.Join(b.DestinationPath, match)); err != nil {
+				return fmt.Errorf("(*sshStorage).Prune: error removing file: %w", err)
+			}
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}

internal/storage/storage.go

@@ -0,0 +1,60 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package storage
+
+import (
+	"time"
+)
+
+// Backend is an interface for defining functions which all storage providers support.
+type Backend interface {
+	Copy(file string) error
+	Prune(deadline time.Time, pruningPrefix string) (*PruneStats, error)
+	Name() string
+}
+
+// StorageBackend is a generic type of storage. Everything here are common properties of all storage types.
+type StorageBackend struct {
+	DestinationPath string
+	RetentionDays   int
+	Log             Log
+}
+
+type LogLevel int
+
+const (
+	LogLevelInfo LogLevel = iota
+	LogLevelWarning
+	LogLevelError
+)
+
+type Log func(logType LogLevel, context string, msg string, params ...any)
+
+// PruneStats is a wrapper struct for returning stats after pruning
+type PruneStats struct {
+	Total  uint
+	Pruned uint
+}
+
+// DoPrune holds general control flow that applies to any kind of storage.
+// Callers can pass in a thunk that performs the actual deletion of files.
+func (b *StorageBackend) DoPrune(context string, lenMatches, lenCandidates int, doRemoveFiles func() error) error {
+	if lenMatches != 0 && lenMatches != lenCandidates {
+		if err := doRemoveFiles(); err != nil {
+			return err
+		}
+		b.Log(LogLevelInfo, context,
+			"Pruned %d out of %d backups as their age exceeded the configured retention period of %d days.",
+			lenMatches,
+			lenCandidates,
+			b.RetentionDays,
+		)
+	} else if lenMatches != 0 && lenMatches == lenCandidates {
+		b.Log(LogLevelWarning, context, "The current configuration would delete all %d existing backups.", lenMatches)
+		b.Log(LogLevelWarning, context, "Refusing to do so, please check your configuration.")
+	} else {
+		b.Log(LogLevelInfo, context, "None of %d existing backups were pruned.", lenCandidates)
+	}
+	return nil
+}

internal/storage/webdav/webdav.go

@@ -0,0 +1,123 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package webdav
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/studio-b12/gowebdav"
+)
+
+type webDavStorage struct {
+	*storage.StorageBackend
+	client *gowebdav.Client
+	url    string
+}
+
+// Config allows to configure a WebDAV storage backend.
+type Config struct {
+	URL         string
+	RemotePath  string
+	Username    string
+	Password    string
+	URLInsecure bool
+}
+
+// NewStorageBackend creates and initializes a new WebDav storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	if opts.Username == "" || opts.Password == "" {
+		return nil, errors.New("NewStorageBackend: WEBDAV_URL is defined, but no credentials were provided")
+	} else {
+		webdavClient := gowebdav.NewClient(opts.URL, opts.Username, opts.Password)
+
+		if opts.URLInsecure {
+			defaultTransport, ok := http.DefaultTransport.(*http.Transport)
+			if !ok {
+				return nil, errors.New("NewStorageBackend: unexpected error when asserting type for http.DefaultTransport")
+			}
+			webdavTransport := defaultTransport.Clone()
+			webdavTransport.TLSClientConfig.InsecureSkipVerify = opts.URLInsecure
+			webdavClient.SetTransport(webdavTransport)
+		}
+
+		return &webDavStorage{
+			StorageBackend: &storage.StorageBackend{
+				DestinationPath: opts.RemotePath,
+				Log:             logFunc,
+			},
+			client: webdavClient,
+		}, nil
+	}
+}
+
+// Name returns the name of the storage backend
+func (b *webDavStorage) Name() string {
+	return "WebDAV"
+}
+
+// Copy copies the given file to the WebDav storage backend.
+func (b *webDavStorage) Copy(file string) error {
+	_, name := path.Split(file)
+	if err := b.client.MkdirAll(b.DestinationPath, 0644); err != nil {
+		return fmt.Errorf("(*webDavStorage).Copy: error creating directory '%s' on server: %w", b.DestinationPath, err)
+	}
+
+	r, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("(*webDavStorage).Copy: error opening the file to be uploaded: %w", err)
+	}
+
+	if err := b.client.WriteStream(filepath.Join(b.DestinationPath, name), r, 0644); err != nil {
+		return fmt.Errorf("(*webDavStorage).Copy: error uploading the file: %w", err)
+	}
+	b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup '%s' to '%s' at path '%s'.", file, b.url, b.DestinationPath)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the WebDav storage backend.
+func (b *webDavStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates, err := b.client.ReadDir(b.DestinationPath)
+	if err != nil {
+		return nil, fmt.Errorf("(*webDavStorage).Prune: error looking up candidates from remote storage: %w", err)
+	}
+	var matches []fs.FileInfo
+	var lenCandidates int
+	for _, candidate := range candidates {
+		if !strings.HasPrefix(candidate.Name(), pruningPrefix) {
+			continue
+		}
+		lenCandidates++
+		if candidate.ModTime().Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(lenCandidates),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), lenCandidates, func() error {
+		for _, match := range matches {
+			if err := b.client.Remove(filepath.Join(b.DestinationPath, match.Name())); err != nil {
+				return fmt.Errorf("(*webDavStorage).Prune: error removing file: %w", err)
+			}
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}

test/Dockerfile

@@ -0,0 +1,13 @@
+FROM docker:24-dind
+
+RUN apk add \
+  coreutils \
+  curl \
+  gpg \
+  jq \
+  moreutils \
+  tar \
+  zstd \
+  --no-cache
+
+WORKDIR /code/test

test/README.md

@@ -0,0 +1,70 @@
+# Integration Tests
+
+## Running tests
+
+The main entry point for running tests is the `./test.sh` script.
+It can be used to run the entire test suite, or just a single test case.
+
+### Run all tests
+
+```sh
+./test.sh
+```
+
+### Run a single test case
+
+```sh
+./test.sh <directory-name>
+```
+
+### Configuring a test run
+
+In addition to the match pattern, which can be given as the first positional argument, certain behavior can be changed by setting environment variables:
+
+#### `BUILD_IMAGE`
+
+When set, the test script will build an up-to-date `docker-volume-backup` image from the current state of your source tree, and run the tests against it.
+
+```sh
+BUILD_IMAGE=1 ./test.sh
+```
+
+The default behavior is not to build an image, and instead look for a version on your host system.
+
+#### `IMAGE_TAG`
+
+Setting this value lets you run tests against different existing images, so you can compare behavior:
+
+```sh
+IMAGE_TAG=v2.30.0 ./test.sh
+```
+
+#### `NO_IMAGE_CACHE`
+
+When set, images from remote registries will not be cached and shared between sandbox containers.
+
+```sh
+NO_IMAGE_CACHE=1 ./test.sh
+```
+
+By default, two local images are created that persist the image data and provide it to containers at runtime.
+
+## Understanding the test setup
+
+The test setup runs each test case in an isolated Docker container, which itself is running an otherwise unused Docker daemon.
+This means, tests can rely on noone else using that daemon, making expectations about the number of running containers and so forth.
+As the sandbox container is also expected to be torn down post test, the scripts do not need to do any clean up or similar.
+
+## Anatomy of a test case
+
+The `test.sh` script looks for an exectuable file called `run.sh` in each directory.
+When found, it is executed and signals success by returning a 0 exit code.
+Any other exit code is considered a failure and will halt execution of further tests.
+
+There is an `util.sh` file containing a few commonly used helpers which can be used by putting the following prelude to a new test case:
+
+```sh
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+```

test/azure/docker-compose.yml

@@ -0,0 +1,56 @@
+version: '3'
+
+services:
+  storage:
+    image: mcr.microsoft.com/azure-storage/azurite:3.26.0
+    volumes:
+      - ${DATA_DIR:-./data}:/data
+    command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data
+    healthcheck:
+      test: nc 127.0.0.1 10000 -z
+      interval: 1s
+      retries: 30
+
+  az_cli:
+    image: mcr.microsoft.com/azure-cli:2.51.0
+    volumes:
+      - ${LOCAL_DIR:-./local}:/dump
+    command:
+      - /bin/sh
+      - -c
+      - |
+          az storage container create --name test-container
+    depends_on:
+      storage:
+        condition: service_healthy
+    environment:
+      AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1;
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    restart: always
+    environment:
+      AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1
+      AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      AZURE_STORAGE_CONTAINER_NAME: test-container
+      AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/
+      AZURE_STORAGE_PATH: 'path/to/backup'
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/azure/run.sh

@@ -0,0 +1,86 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
+export DATA_DIR=$(mktemp -d)
+
+download_az () {
+  docker compose run --rm az_cli \
+    az storage blob download -f /dump/$1.tar.gz -c test-container -n path/to/backup/$1.tar.gz
+}
+
+docker compose up -d --quiet-pull
+
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+download_az "test"
+
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR
+
+if [ ! -f "$TMP_DIR/backup/app_data/offen.db" ]; then
+  fail "Could not find expeced file in untared backup"
+fi
+
+pass "Found relevant files in untared remote backups."
+rm "$LOCAL_DIR/test.tar.gz"
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+download_az "test"
+if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then
+  fail "Remote backup was deleted"
+fi
+pass "Remote backups have not been deleted."
+
+# The third part of this test checks if old backups get deleted when the retention
+# is set to 7 days (which it should)
+
+BACKUP_RETENTION_DAYS="7" docker compose up -d
+sleep 5
+
+info "Create first backup with no prune"
+docker compose exec backup backup
+
+docker compose run --rm az_cli \
+    az storage blob upload -f /dump/test.tar.gz -c test-container -n path/to/backup/test-old.tar.gz
+
+docker compose down
+rm "$LOCAL_DIR/test.tar.gz"
+
+back_date="$(date "+%Y-%m-%dT%H:%M:%S%z" -d "14 days ago" | rev | cut -c 3- | rev):00"
+jq --arg back_date "$back_date" '(.collections[] | select(.name=="$BLOBS_COLLECTION$") | .data[] | select(.name=="path/to/backup/test-old.tar.gz") | .properties.creationTime = $back_date)' "$DATA_DIR/__azurite_db_blob__.json" | sponge "$DATA_DIR/__azurite_db_blob__.json"
+
+docker compose up -d
+sleep 5
+
+info "Create second backup and prune"
+docker compose exec backup backup
+
+info "Download first backup which should be pruned"
+download_az "test-old" || true
+if [ -f "$LOCAL_DIR/test-old.tar.gz" ]; then
+  fail "Backdated file was not deleted"
+fi
+download_az "test" || true
+if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then
+  fail "Recent file was not found"
+fi
+
+pass "Old remote backup has been pruned, new one is still present."

test/certs/docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3'
+
+services:
+  minio:
+    hostname: minio.local
+    image: minio/minio:RELEASE.2020-08-04T23-10-51Z
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server --certs-dir "/certs" --address ":443" /data'
+    volumes:
+      - minio_backup_data:/data
+      - ${CERT_DIR:-.}/minio.crt:/certs/public.crt
+      - ${CERT_DIR:-.}/minio.key:/certs/private.key
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    depends_on:
+      - minio
+    restart: always
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ
+      AWS_ENDPOINT: minio.local:443
+      AWS_ENDPOINT_CA_CERT: /root/minio-rootCA.crt
+      AWS_S3_BUCKET_NAME: backup
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${CERT_DIR:-.}/rootCA.crt:/root/minio-rootCA.crt
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  minio_backup_data:
+    name: minio_backup_data
+  app_data:

test/certs/run.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export CERT_DIR=$(mktemp -d)
+
+openssl genrsa -des3 -passout pass:test -out "$CERT_DIR/rootCA.key" 4096
+openssl req -passin pass:test \
+  -subj "/C=DE/ST=BE/O=IntegrationTest, Inc." \
+  -x509 -new -key "$CERT_DIR/rootCA.key" -sha256 -days 1 -out "$CERT_DIR/rootCA.crt"
+
+openssl genrsa -out "$CERT_DIR/minio.key" 4096
+openssl req -new -sha256 -key "$CERT_DIR/minio.key" \
+  -subj "/C=DE/ST=BE/O=IntegrationTest, Inc./CN=minio" \
+  -out "$CERT_DIR/minio.csr"
+
+openssl x509 -req -passin pass:test \
+  -in "$CERT_DIR/minio.csr" \
+  -CA "$CERT_DIR/rootCA.crt" -CAkey "$CERT_DIR/rootCA.key" -CAcreateserial \
+  -extfile san.cnf \
+  -out "$CERT_DIR/minio.crt" -days 1 -sha256
+
+openssl x509 -in "$CERT_DIR/minio.crt" -noout -text
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'tar -xvf /minio_data/backup/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in untared remote backups."

test/certs/san.cnf

@@ -0,0 +1 @@
+subjectAltName = DNS:minio.local

test/cli/run.sh

@@ -3,12 +3,17 @@
 set -e
 
 cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
 
 docker network create test_network
 docker volume create backup_data
 docker volume create app_data
+# This volume is created to test whether empty directories are handled
+# correctly. It is not supposed to hold any data.
+docker volume create empty_data
 
-docker run -d \
+docker run -d -q \
   --name minio \
   --network test_network \
   --env MINIO_ROOT_USER=test \
@@ -20,7 +25,7 @@ docker run -d \
 
 docker exec minio mkdir -p /data/backup
 
-docker run -d \
+docker run -d -q \
   --name offen \
   --network test_network \
   -v app_data:/var/opt/offen/ \
@@ -28,9 +33,10 @@ docker run -d \
 
 sleep 10
 
-docker run --rm \
+docker run --rm -q \
   --network test_network \
   -v app_data:/backup/app_data \
+  -v empty_data:/backup/empty_data \
   -v /var/run/docker.sock:/var/run/docker.sock \
   --env AWS_ACCESS_KEY_ID=test \
   --env AWS_SECRET_ACCESS_KEY=GMusLtUmILge2by+z890kQ \
@@ -40,21 +46,17 @@ docker run --rm \
   --env BACKUP_FILENAME=test.tar.gz \
   --env "BACKUP_FROM_SNAPSHOT=true" \
   --entrypoint backup \
-  offen/docker-volume-backup:$TEST_VERSION
+  offen/docker-volume-backup:${TEST_VERSION:-canary}
 
-docker run --rm -it \
+docker run --rm -q \
   -v backup_data:/data alpine \
-  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
+  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db && test -d /backup/empty_data'
 
-echo "[TEST:PASS] Found relevant files in untared backup."
+pass "Found relevant files in untared remote backup."
 
-if [ "$(docker ps -q | wc -l)" != "2" ]; then
+# This test does not stop containers during backup. This is happening on
-  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
+# purpose in order to cover this setup as well.
-  docker ps
+expect_running_containers "2"
-  exit 1
-fi
-
-echo "[TEST:PASS] All containers running post backup."
 
 docker rm $(docker stop minio offen)
 docker volume rm backup_data app_data

test/commands/docker-compose.yml

@@ -0,0 +1,50 @@
+version: '3.8'
+
+services:
+  database:
+    image: mariadb:10.7
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: backup
+    labels:
+      # this is testing the deprecated label on purpose
+      - docker-volume-backup.exec-pre=/bin/sh -c 'mysqldump -ptest --all-databases > /tmp/volume/dump.sql'
+      - docker-volume-backup.copy-post=/bin/sh -c 'echo "post" > /tmp/volume/post.txt'
+      - docker-volume-backup.exec-label=test
+    volumes:
+      - app_data:/tmp/volume
+
+  other_database:
+    image: mariadb:10.7
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: backup
+    labels:
+      - docker-volume-backup.archive-pre=touch /tmp/volume/not-relevant.txt
+      - docker-volume-backup.exec-label=not-relevant
+    volumes:
+      - app_data:/tmp/volume
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      EXEC_LABEL: test
+      EXEC_FORWARD_OUTPUT: "true"
+    volumes:
+      - ${LOCAL_DIR:-./local}:/archive
+      - app_data:/backup/data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  app_data:

test/commands/run.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 30 # mariadb likes to take a bit before responding
+
+docker compose exec backup backup
+
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR
+if [ ! -f "$TMP_DIR/backup/data/dump.sql" ]; then
+  fail "Could not find file written by pre command."
+fi
+pass "Found expected file."
+
+if [ -f "$TMP_DIR/backup/data/not-relevant.txt" ]; then
+  fail "Command ran for container with other label."
+fi
+pass "Command did not run for container with other label."
+
+if [ -f "$TMP_DIR/backup/data/post.txt" ]; then
+  fail "File created in post command was present in backup."
+fi
+pass "Did not find unexpected file."
+
+docker compose down --volumes
+
+info "Running commands test in swarm mode next."
+
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
+
+docker swarm init
+
+docker stack deploy --compose-file=docker-compose.yml test_stack
+
+while [ -z $(docker ps -q -f name=backup) ]; do
+  info "Backup container not ready yet. Retrying."
+  sleep 1
+done
+
+sleep 20
+
+docker exec $(docker ps -q -f name=backup) backup
+
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR
+if [ ! -f "$TMP_DIR/backup/data/dump.sql" ]; then
+  fail "Could not find file written by pre command."
+fi
+pass "Found expected file."
+
+if [ -f "$TMP_DIR/backup/data/post.txt" ]; then
+  fail "File created in post command was present in backup."
+fi
+pass "Did not find unexpected file."

test/compose/.gitignore

@@ -1 +0,0 @@
-local

test/compose/run.sh

@@ -1,58 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(dirname $0)
-
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-
-docker-compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link
-docker-compose exec backup backup
-
-docker run --rm -it \
-  -v compose_backup_data:/data alpine \
-  ash -c 'apk add gnupg && echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /data/backup/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
-
-echo "[TEST:PASS] Found relevant files in untared remote backup."
-
-test -L ./local/test-hostnametoken.latest.tar.gz.gpg
-echo 1234secret | gpg -d --yes --passphrase-fd 0 ./local/test-hostnametoken.tar.gz.gpg > ./local/decrypted.tar.gz
-tar -xf ./local/decrypted.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
-rm ./local/decrypted.tar.gz
-test -L /tmp/backup/app_data/db.link
-
-echo "[TEST:PASS] Found relevant files in untared local backup."
-
-if [ "$(docker-compose ps -q | wc -l)" != "3" ]; then
-  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
-  docker-compose ps
-  exit 1
-fi
-
-echo "[TEST:PASS] All containers running post backup."
-
-# The second part of this test checks if backups get deleted when the retention
-# is set to 0 days (which it should not as it would mean all backups get deleted)
-# TODO: find out if we can test actual deletion without having to wait for a day
-BACKUP_RETENTION_DAYS="0" docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-docker run --rm -it \
-  -v compose_backup_data:/data alpine \
-  ash -c '[ $(find /data/backup/ -type f | wc -l) = "1" ]'
-
-echo "[TEST:PASS] Remote backups have not been deleted."
-
-if [ "$(find ./local -type f | wc -l)" != "1" ]; then
-  echo "[TEST:FAIL] Backups should not have been deleted, instead seen:"
-  find ./local -type f
-fi
-
-echo "[TEST:PASS] Local backups have not been deleted."
-
-docker-compose down --volumes

test/confd/01backup.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="conf.tar.gz"
+BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/02backup.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="other.tar.gz"
+BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/03never.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="never.tar.gz"
+BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?"

test/confd/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    volumes:
+      - ${LOCAL_DIR:-./local}:/archive
+      - app_data:/backup/app_data:ro
+      - ./01backup.env:/etc/dockervolumebackup/conf.d/01backup.env
+      - ./02backup.env:/etc/dockervolumebackup/conf.d/02backup.env
+      - ./03never.env:/etc/dockervolumebackup/conf.d/03never.env
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/confd/run.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+
+# sleep until a backup is guaranteed to have happened on the 1 minute schedule
+sleep 100
+
+if [ ! -f "$LOCAL_DIR/conf.tar.gz" ]; then
+  fail "Config from file was not used."
+fi
+pass "Config from file was used."
+
+if [ ! -f "$LOCAL_DIR/other.tar.gz" ]; then
+  fail "Run on same schedule did not succeed."
+fi
+pass "Run on same schedule succeeded."
+
+if [ -f "$LOCAL_DIR/never.tar.gz" ]; then
+  fail "Unexpected file was found."
+fi
+pass "Unexpected cron did not run."

test/dropbox/.gitignore

@@ -0,0 +1 @@
+user_v2_ready.yaml

test/dropbox/docker-compose.yml

@@ -0,0 +1,57 @@
+version: '3'
+
+services:
+  openapi_mock:
+    image: muonsoft/openapi-mock:0.3.9
+    environment:
+      OPENAPI_MOCK_USE_EXAMPLES: if_present
+      OPENAPI_MOCK_SPECIFICATION_URL: '/etc/openapi/user_v2.yaml'
+    ports:
+      - 8080:8080
+    volumes:
+      - ${SPEC_FILE:-./user_v2.yaml}:/etc/openapi/user_v2.yaml
+
+  oauth2_mock:
+    image: ghcr.io/navikt/mock-oauth2-server:1.0.0
+    ports:
+      - 8090:8090
+    environment:
+      PORT: 8090
+      JSON_CONFIG_PATH: '/etc/oauth2/config.json'
+    volumes:
+      - ./oauth2_config.json:/etc/oauth2/config.json
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    depends_on:
+      - openapi_mock
+      - oauth2_mock
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      DROPBOX_ENDPOINT: http://openapi_mock:8080
+      DROPBOX_OAUTH2_ENDPOINT: http://oauth2_mock:8090
+      DROPBOX_REFRESH_TOKEN: test
+      DROPBOX_APP_KEY: test
+      DROPBOX_APP_SECRET: test
+      DROPBOX_REMOTE_PATH: /test
+      DROPBOX_CONCURRENCY_LEVEL: 6
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/dropbox/oauth2_config.json

@@ -0,0 +1,37 @@
+{
+   "interactiveLogin": true,
+   "httpServer": "NettyWrapper",
+   "tokenCallbacks": [
+       {
+           "issuerId": "issuer1",
+           "tokenExpiry": 120,
+           "requestMappings": [
+               {
+                   "requestParam": "scope",
+                   "match": "scope1",
+                   "claims": {
+                       "sub": "subByScope",
+                       "aud": [
+                           "audByScope"
+                       ]
+                   }
+               }
+           ]
+       },
+       {
+           "issuerId": "issuer2",
+           "requestMappings": [
+               {
+                   "requestParam": "someparam",
+                   "match": "somevalue",
+                   "claims": {
+                       "sub": "subBySomeParam",
+                       "aud": [
+                           "audBySomeParam"
+                       ]
+                   }
+               }
+           ]
+       }
+   ]
+}

test/dropbox/run.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export SPEC_FILE=$(mktemp -d)/user_v2.yaml
+cp user_v2.yaml $SPEC_FILE
+sed -i 's/SERVER_MODIFIED_1/'"$(date "+%Y-%m-%dT%H:%M:%SZ")/g" $SPEC_FILE
+sed -i 's/SERVER_MODIFIED_2/'"$(date "+%Y-%m-%dT%H:%M:%SZ" -d "14 days ago")/g" $SPEC_FILE
+
+docker compose up -d --quiet-pull
+sleep 5
+
+logs=$(docker compose exec -T backup backup)
+
+sleep 5
+
+expect_running_containers "4"
+
+echo "$logs"
+if echo "$logs" | grep -q "ERROR"; then
+  fail "Backup failed, errors reported: $logs"
+else
+  pass "Backup succeeded, no errors reported."
+fi
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+logs=$(docker compose exec -T backup backup)
+
+echo "$logs"
+if echo "$logs" | grep -q "Refusing to do so, please check your configuration"; then
+  pass "Remote backups have not been deleted."
+else
+  fail "Remote backups would have been deleted: $logs"
+fi
+
+# The third part of this test checks if old backups get deleted when the retention
+# is set to 7 days (which it should)
+BACKUP_RETENTION_DAYS="7" docker compose up -d
+sleep 5
+
+info "Create second backup and prune"
+logs=$(docker compose exec -T backup backup)
+
+echo "$logs"
+if echo "$logs" | grep -q "Pruned 1 out of 2 backups as their age exceeded the configured retention period"; then
+  pass "Old remote backup has been pruned, new one is still present."
+elif echo "$logs" | grep -q "ERROR"; then
+  fail "Pruning failed, errors reported: $logs"
+elif echo "$logs" | grep -q "None of 1 existing backups were pruned"; then
+  fail "Pruning failed, old backup has not been pruned: $logs"
+else
+  fail "Pruning failed, unknown result: $logs"
+fi

test/extend/Dockerfile

@@ -0,0 +1,4 @@
+ARG version=canary
+FROM offen/docker-volume-backup:$version
+
+RUN apk add rsync

test/extend/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    labels:
+      - docker-volume-backup.copy-post=/bin/sh -c 'mkdir -p /tmp/unpack && tar -xvf $$COMMAND_RUNTIME_ARCHIVE_FILEPATH -C /tmp/unpack && rsync -r /tmp/unpack/backup/app_data /local'
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      EXEC_FORWARD_OUTPUT: "true"
+    volumes:
+      - ${LOCAL_DIR:-local}:/local
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/extend/run.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+export BASE_VERSION="${TEST_VERSION:-canary}"
+export TEST_VERSION="${TEST_VERSION:-canary}-with-rsync"
+
+docker build . -t offen/docker-volume-backup:$TEST_VERSION --build-arg version=$BASE_VERSION
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "2"
+
+if [ ! -f "$LOCAL_DIR/app_data/offen.db" ]; then
+  fail "Could not find expected file in untared archive."
+fi

test/gpg/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    environment:
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      GPG_PASSPHRASE: 1234#$$ecret
+    volumes:
+      - ${LOCAL_DIR:-./local}:/archive
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/gpg/run.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+expect_running_containers "2"
+
+TMP_DIR=$(mktemp -d)
+
+echo "1234#\$ecret" | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 "$LOCAL_DIR/test.tar.gz.gpg" > "$LOCAL_DIR/decrypted.tar.gz"
+tar -xf "$LOCAL_DIR/decrypted.tar.gz" -C $TMP_DIR
+
+if [ ! -f $TMP_DIR/backup/app_data/offen.db ]; then
+  fail "Could not find expected file in untared archive."
+fi
+rm "$LOCAL_DIR/decrypted.tar.gz"
+
+pass "Found relevant files in decrypted and untared local backup."
+
+if [ ! -L "$LOCAL_DIR/test-latest.tar.gz.gpg" ]; then
+  fail "Could not find local symlink to latest encrypted backup."
+fi

test/ignore/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_EXCLUDE_REGEXP: '\.(me|you)$$'
+    volumes:
+      - ${LOCAL_DIR:-./local}:/archive
+      - ./sources:/backup/data:ro

test/ignore/run.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+docker compose exec backup backup
+
+TMP_DIR=$(mktemp -d)
+tar --same-owner -xvf "$LOCAL_DIR/test.tar.gz" -C "$TMP_DIR"
+
+if [ ! -f "$TMP_DIR/backup/data/me.txt" ]; then
+  fail "Expected file was not found."
+fi
+pass "Expected file was found."
+
+if [ -f "$TMP_DIR/backup/data/skip.me" ]; then
+  fail "Ignored file was found."
+fi
+pass "Ignored file was not found."

test/local/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${LOCAL_DIR:-./local}:/archive
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/local/run.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+
+# A symlink for a known file in the volume is created so the test can check
+# whether symlinks are preserved on backup.
+docker compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "2"
+
+tmp_dir=$(mktemp -d)
+tar -xvf "$LOCAL_DIR/test-hostnametoken.tar.gz" -C $tmp_dir
+if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then
+  fail "Could not find expected file in untared archive."
+fi
+rm -f "$LOCAL_DIR/test-hostnametoken.tar.gz"
+
+if [ ! -L "$tmp_dir/backup/app_data/db.link" ]; then
+  fail "Could not find expected symlink in untared archive."
+fi
+
+pass "Found relevant files in decrypted and untared local backup."
+
+if [ ! -L "$LOCAL_DIR/test-hostnametoken.latest.tar.gz.gpg" ]; then
+  fail "Could not find symlink to latest version."
+fi
+
+pass "Found symlink to latest version in local backup."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+if [ "$(find "$LOCAL_DIR" -type f | wc -l)" != "1" ]; then
+  fail "Backups should not have been deleted, instead seen: "$(find "$local_dir" -type f)""
+fi
+pass "Local backups have not been deleted."
+
+# The third part of this test checks if old backups get deleted when the retention
+# is set to 7 days (which it should)
+
+BACKUP_RETENTION_DAYS="7" docker compose up -d
+sleep 5
+
+info "Create first backup with no prune"
+docker compose exec backup backup
+
+touch -r "$LOCAL_DIR/test-hostnametoken.tar.gz" -d "14 days ago" "$LOCAL_DIR/test-hostnametoken-old.tar.gz"
+
+info "Create second backup and prune"
+docker compose exec backup backup
+
+if [ -f "$LOCAL_DIR/test-hostnametoken-old.tar.gz" ]; then
+  fail "Backdated file has not been deleted."
+fi
+
+if [ ! -f "$LOCAL_DIR/test-hostnametoken.tar.gz" ]; then
+  fail "Recent file has been deleted."
+fi
+
+pass "Old remote backup has been pruned, new one is still present."

test/notifications/docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_PRUNING_PREFIX: test
+      NOTIFICATION_LEVEL: info
+      NOTIFICATION_URLS: ${NOTIFICATION_URLS}
+      EXTRA_VALUE: extra-value
+    volumes:
+      - ${LOCAL_DIR:-./local}:/archive
+      - app_data:/backup/app_data:ro
+      - ./notifications.tmpl:/etc/dockervolumebackup/notifications.d/notifications.tmpl
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+  gotify:
+    image: gotify/server
+    ports:
+      - 8080:80
+    environment:
+      - GOTIFY_DEFAULTUSER_PASS=custom
+    volumes:
+      - gotify_data:/app/data
+
+volumes:
+  app_data:
+  gotify_data:

test/notifications/notifications.tmpl

@@ -0,0 +1,7 @@
+{{ define "title_success" -}}
+Successful test run with {{ env "EXTRA_VALUE" }}, yay!
+{{- end }}
+
+{{ define "body_success" -}}
+Backing up {{ .Stats.BackupFile.FullPath }} succeeded.
+{{- end }}

test/notifications/run.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+
+GOTIFY_TOKEN=$(curl -sSLX POST -H 'Content-Type: application/json' -d '{"name":"test"}' http://admin:custom@localhost:8080/application | jq -r '.token')
+info "Set up Gotify application using token $GOTIFY_TOKEN"
+
+docker compose exec backup backup
+
+NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
+if [ "$NUM_MESSAGES" != 0 ]; then
+  fail "Expected no notifications to be sent when not configured"
+fi
+pass "No notifications were sent when not configured."
+
+docker compose down
+
+NOTIFICATION_URLS="gotify://gotify/${GOTIFY_TOKEN}?disableTLS=true" docker compose up -d
+
+docker compose exec backup backup
+
+NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
+if [ "$NUM_MESSAGES" != 1 ]; then
+  fail "Expected one notifications to be sent when configured"
+fi
+pass "Correct number of notifications were sent when configured."
+
+MESSAGE_TITLE=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].title')
+MESSAGE_BODY=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].message')
+
+if [ "$MESSAGE_TITLE" != "Successful test run with extra-value, yay!" ]; then
+  fail "Unexpected notification title $MESSAGE_TITLE"
+fi
+pass "Custom notification title was used."
+
+if [ "$MESSAGE_BODY" != "Backing up /tmp/test.tar.gz succeeded." ]; then
+  fail "Unexpected notification body $MESSAGE_BODY"
+fi
+pass "Custom notification body was used."

test/ownership/docker-compose.yml

@@ -0,0 +1,27 @@
+version: '3'
+
+services:
+  db:
+    image: postgres:14-alpine
+    restart: unless-stopped
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: 1FHJMSwt0yhIN1zS7I4DilGUhThBKq0x
+      POSTGRES_USER: test
+      POSTGRES_DB: test
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION}
+    restart: always
+    environment:
+      BACKUP_FILENAME: backup.tar.gz
+    volumes:
+      - postgres_data:/backup/postgres:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ${LOCAL_DIR:-./local}:/archive
+
+volumes:
+  postgres_data:

test/ownership/run.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+# This test refers to https://github.com/offen/docker-volume-backup/issues/71
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+TMP_DIR=$(mktemp -d)
+tar --same-owner -xvf "$LOCAL_DIR/backup.tar.gz" -C $TMP_DIR
+
+find $TMP_DIR/backup/postgres > /dev/null
+pass "Backup contains files at expected location"
+
+for file in $(find $TMP_DIR/backup/postgres); do
+  if [ "$(stat -c '%u:%g' $file)" != "70:70" ]; then
+    fail "Unexpected file ownership for $file: $(stat -c '%u:%g' $file)"
+  fi
+done
+pass "All files and directories in backup preserved their ownership."

test/pgzip/run.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker network create test_network
+docker volume create app_data
+
+LOCAL_DIR=$(mktemp -d)
+
+docker run -d -q \
+  --name offen \
+  --network test_network \
+  -v app_data:/var/opt/offen/ \
+  offen/offen:latest
+
+sleep 5
+
+docker run --rm -q \
+  --network test_network \
+  -v app_data:/backup/app_data \
+  -v $LOCAL_DIR:/archive \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  --env BACKUP_COMPRESSION=gz \
+  --env GZIP_PARALLELISM=0 \
+  --env BACKUP_FILENAME='test.{{ .Extension }}' \
+  --entrypoint backup \
+  offen/docker-volume-backup:${TEST_VERSION:-canary}
+
+tmp_dir=$(mktemp -d)
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C $tmp_dir
+if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then
+  fail "Could not find expected file in untared archive."
+fi
+pass "Found relevant files in untared local backup."
+
+# This test does not stop containers during backup. This is happening on
+# purpose in order to cover this setup as well.
+expect_running_containers "1"

test/pruning/docker-compose.yml

@@ -0,0 +1,50 @@
+version: '3'
+
+services:
+  minio:
+    image: minio/minio:RELEASE.2020-08-04T23-10-51Z
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
+    volumes:
+      - minio_backup_data:/data
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    depends_on:
+      - minio
+    restart: always
+    environment:
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ
+      AWS_ENDPOINT: minio:9000
+      AWS_ENDPOINT_PROTO: http
+      AWS_S3_BUCKET_NAME: backup
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: 7
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz
+      BACKUP_SKIP_BACKENDS_FROM_PRUNE: 's3'
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./local:/archive
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:
+  minio_backup_data:
+    name: minio_backup_data

test/pruning/run.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+
+# Tests prune-skipping with multiple backends (local, s3)
+# Pruning itself is tested individually for each storage backend
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+touch -r ./local/test-hostnametoken.tar.gz -d "14 days ago" ./local/test-hostnametoken-old.tar.gz
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'touch -d@$(( $(date +%s) - 1209600 )) /minio_data/backup/test-hostnametoken-old.tar.gz'
+
+# Skip s3 backend from prune
+
+docker compose up -d
+sleep 5
+
+info "Create backup with no prune for s3 backend"
+docker compose exec backup backup
+
+info "Check if old backup has been pruned (local)"
+test ! -f ./local/test-hostnametoken-old.tar.gz
+
+info "Check if old backup has NOT been pruned (s3)"
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'test -f /minio_data/backup/test-hostnametoken-old.tar.gz'
+
+pass "Old remote backup has been pruned locally, skipped S3 backend is untouched."
+
+# Skip local and s3 backend from prune (all backends)
+
+touch -r ./local/test-hostnametoken.tar.gz -d "14 days ago" ./local/test-hostnametoken-old.tar.gz
+
+docker compose up -d
+sleep 5
+
+info "Create backup with no prune for both backends"
+docker compose exec -e BACKUP_SKIP_BACKENDS_FROM_PRUNE="s3,local" backup backup
+
+info "Check if old backup has NOT been pruned (local)"
+if [ ! -f ./local/test-hostnametoken-old.tar.gz ]; then
+  fail "Backdated file has not been deleted"
+fi
+
+info "Check if old backup has NOT been pruned (s3)"
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'test -f /minio_data/backup/test-hostnametoken-old.tar.gz'
+
+pass "Skipped all backends while pruning."

test/s3/docker-compose.yml

@@ -10,10 +10,10 @@ services:
       MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
     entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
     volumes:
-      - backup_data:/data
+      - minio_backup_data:/data
 
-  backup: &default_backup_service
+  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION}
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
     hostname: hostnametoken
     depends_on:
       - minio
@@ -26,14 +26,11 @@ services:
       AWS_S3_BUCKET_NAME: backup
       BACKUP_FILENAME_EXPAND: 'true'
       BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
-      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
-      GPG_PASSPHRASE: 1234secret
     volumes:
-      - ./local:/archive
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 
@@ -45,5 +42,6 @@ services:
       - app_data:/var/opt/offen
 
 volumes:
-  backup_data:
+  minio_backup_data:
+    name: minio_backup_data
   app_data:

test/s3/run.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'tar -xvf /minio_data/backup/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in untared remote backups."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c '[ $(find /minio_data/backup/ -type f | wc -l) = "1" ]'
+
+pass "Remote backups have not been deleted."
+
+# The third part of this test checks if old backups get deleted when the retention
+# is set to 7 days (which it should)
+
+BACKUP_RETENTION_DAYS="7" docker compose up -d
+sleep 5
+
+info "Create first backup with no prune"
+docker compose exec backup backup
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'touch -d@$(( $(date +%s) - 1209600 )) /minio_data/backup/test-hostnametoken-old.tar.gz'
+
+info "Create second backup and prune"
+docker compose exec backup backup
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'test ! -f /minio_data/backup/test-hostnametoken-old.tar.gz && test -f /minio_data/backup/test-hostnametoken.tar.gz'
+
+pass "Old remote backup has been pruned, new one is still present."

test/secrets/docker-compose.yml

@@ -0,0 +1,78 @@
+# Copyright 2020-2021 - Offen Authors <hioffen@posteo.de>
+# SPDX-License-Identifier: Unlicense
+
+version: '3.8'
+
+services:
+  minio:
+    image: minio/minio:RELEASE.2020-08-04T23-10-51Z
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
+    volumes:
+      - backup_data:/data
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    depends_on:
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_root_user
+      AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_root_password
+      AWS_ENDPOINT: minio:9000
+      AWS_ENDPOINT_PROTO: http
+      AWS_S3_BUCKET_NAME: backup
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: 7
+      BACKUP_PRUNING_LEEWAY: 5s
+    volumes:
+      - pg_data:/backup/pg_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    secrets:
+      - minio_root_user
+      - minio_root_password
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    healthcheck:
+      disable: true
+    deploy:
+      replicas: 2
+      restart_policy:
+        condition: on-failure
+
+  pg:
+    image: postgres:14-alpine
+    environment:
+      POSTGRES_PASSWORD: example
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    deploy:
+      restart_policy:
+        condition: on-failure
+
+volumes:
+  backup_data:
+    name: backup_data
+  pg_data:
+    name: pg_data
+
+secrets:
+  minio_root_user:
+    external: true
+  minio_root_password:
+    external: true

test/secrets/run.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker swarm init
+
+printf "test" | docker secret create minio_root_user -
+printf "GMusLtUmILge2by+z890kQ" | docker secret create minio_root_password -
+
+docker stack deploy --compose-file=docker-compose.yml test_stack
+
+while [ -z $(docker ps -q -f name=backup) ]; do
+  info "Backup container not ready yet. Retrying."
+  sleep 1
+done
+
+sleep 20
+
+docker exec $(docker ps -q -f name=backup) backup
+
+docker run --rm \
+  -v backup_data:/data alpine \
+  ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
+
+pass "Found relevant files in untared backup."
+
+sleep 5
+expect_running_containers "5"
+
+docker exec -e AWS_ACCESS_KEY_ID=test $(docker ps -q -f name=backup) backup \
+  && fail "Backup should have failed due to duplicate env variables."
+
+pass "Backup failed due to duplicate env variables."
+
+docker exec -e AWS_ACCESS_KEY_ID_FILE=/tmp/nonexistant $(docker ps -q -f name=backup) backup \
+  && fail "Backup should have failed due to non existing file env variable."
+
+pass "Backup failed due to non existing file env variable."

test/ssh/docker-compose.yml

@@ -0,0 +1,47 @@
+version: '3'
+
+services:
+  ssh:
+    image: linuxserver/openssh-server:version-8.6_p1-r3
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - USER_NAME=test
+    volumes:
+      - ${KEY_DIR:-.}/id_rsa.pub:/config/.ssh/authorized_keys
+      - ssh_backup_data:/tmp
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    depends_on:
+      - ssh
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      SSH_HOST_NAME: ssh
+      SSH_PORT: 2222
+      SSH_USER: test
+      SSH_REMOTE_PATH: /tmp
+      SSH_IDENTITY_PASSPHRASE: test1234
+    volumes:
+      - ${KEY_DIR:-.}/id_rsa:/root/.ssh/id_rsa
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  ssh_backup_data:
+    name: ssh_backup_data
+  app_data:

test/ssh/run.sh

@@ -0,0 +1,67 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export KEY_DIR=$(mktemp -d)
+
+ssh-keygen -t rsa -m pem -b 4096 -N "test1234" -f "$KEY_DIR/id_rsa" -C "docker-volume-backup@local"
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers 3
+
+docker run --rm \
+  -v ssh_backup_data:/ssh_data \
+  alpine \
+  ash -c 'tar -xvf /ssh_data/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in decrypted and untared remote backups."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+docker run --rm \
+  -v ssh_backup_data:/ssh_data \
+  alpine \
+  ash -c '[ $(find /ssh_data/ -type f | wc -l) = "1" ]'
+
+pass "Remote backups have not been deleted."
+
+# The third part of this test checks if old backups get deleted when the retention
+# is set to 7 days (which it should)
+
+BACKUP_RETENTION_DAYS="7" docker compose up -d
+sleep 5
+
+info "Create first backup with no prune"
+docker compose exec backup backup
+
+# Set the modification date of the old backup to 14 days ago
+docker run --rm \
+  -v ssh_backup_data:/ssh_data \
+  --user 1000 \
+  alpine \
+  ash -c 'touch -d@$(( $(date +%s) - 1209600 )) /ssh_data/test-hostnametoken-old.tar.gz'
+
+info "Create second backup and prune"
+docker compose exec backup backup
+
+docker run --rm \
+  -v ssh_backup_data:/ssh_data \
+  alpine \
+  ash -c 'test ! -f /ssh_data/test-hostnametoken-old.tar.gz && test -f /ssh_data/test-hostnametoken.tar.gz'
+
+pass "Old remote backup has been pruned, new one is still present."

test/swarm/docker-compose.yml

@@ -18,8 +18,8 @@ services:
     volumes:
       - backup_data:/data
 
-  backup: &default_backup_service
+  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION}
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
     depends_on:
       - minio
     deploy:
@@ -43,13 +43,15 @@ services:
     image: offen/offen:latest
     labels:
       - docker-volume-backup.stop-during-backup=true
+    healthcheck:
+      disable: true
     deploy:
       replicas: 2
       restart_policy:
         condition: on-failure
 
   pg:
-    image: postgres:12.2-alpine
+    image: postgres:14-alpine
     environment:
       POSTGRES_PASSWORD: example
     labels:
@@ -62,4 +64,6 @@ services:
 
 volumes:
   backup_data:
+    name: backup_data
   pg_data:
+    name: pg_data

test/swarm/run.sh

@@ -3,13 +3,15 @@
 set -e
 
 cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
 
 docker swarm init
 
 docker stack deploy --compose-file=docker-compose.yml test_stack
 
 while [ -z $(docker ps -q -f name=backup) ]; do
-  echo "[TEST:INFO] Backup container not ready yet. Retrying."
+  info "Backup container not ready yet. Retrying."
   sleep 1
 done
 
@@ -17,20 +19,11 @@ sleep 20
 
 docker exec $(docker ps -q -f name=backup) backup
 
-docker run --rm -it \
+docker run --rm \
-  -v test_stack_backup_data:/data alpine \
+  -v backup_data:/data alpine \
   ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
 
-echo "[TEST:PASS] Found relevant files in untared backup."
+pass "Found relevant files in untared backup."
 
-if [ "$(docker ps -q | wc -l)" != "5" ]; then
+sleep 5
-  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
+expect_running_containers "5"
-  docker ps -a
-  exit 1
-fi
-
-echo "[TEST:PASS] All containers running post backup."
-
-docker stack rm test_stack
-
-docker swarm leave --force

test/test.sh

@@ -2,15 +2,69 @@
 
 set -e
 
-TEST_VERSION=${1:-canary}
+MATCH_PATTERN=$1
+IMAGE_TAG=${IMAGE_TAG:-canary}
 
-for dir in $(ls -d -- */); do
+sandbox="docker_volume_backup_test_sandbox"
-  test="${dir}run.sh"
+tarball="$(mktemp -d)/image.tar.gz"
+
+trap finish EXIT INT TERM
+
+finish () {
+  rm -rf $(dirname $tarball)
+  if [ ! -z $(docker ps -aq --filter=name=$sandbox) ]; then
+    docker rm -f $(docker stop $sandbox)
+  fi
+  if [ ! -z $(docker volume ls -q --filter=name="^${sandbox}\$") ]; then
+    docker volume rm $sandbox
+  fi
+}
+
+docker build -t offen/docker-volume-backup:test-sandbox .
+
+if [ ! -z "$BUILD_IMAGE" ]; then
+  docker build -t offen/docker-volume-backup:$IMAGE_TAG $(dirname $(pwd))
+fi
+
+docker save offen/docker-volume-backup:$IMAGE_TAG -o $tarball
+
+find_args="-mindepth 1 -maxdepth 1 -type d"
+if [ ! -z "$MATCH_PATTERN" ]; then
+  find_args="$find_args -name $MATCH_PATTERN"
+fi
+
+for dir in $(find $find_args | sort); do
+  dir=$(echo $dir | cut -c 3-)
   echo "################################################"
-  echo "Now running $test"
+  echo "Now running ${dir}"
   echo "################################################"
   echo ""
-  TEST_VERSION=$TEST_VERSION /bin/sh $test
+
+  test="${dir}/run.sh"
+  docker_run_args="--name "$sandbox" --detach \
+    --privileged \
+    -v $(dirname $(pwd)):/code \
+    -v $tarball:/cache/image.tar.gz \
+    -v $sandbox:/var/lib/docker"
+
+  if [ -z "$NO_IMAGE_CACHE" ]; then
+    docker_run_args="$docker_run_args \
+    -v "${sandbox}_image":/var/lib/docker/image \
+    -v "${sandbox}_overlay2":/var/lib/docker/overlay2"
+  fi
+
+  docker run $docker_run_args offen/docker-volume-backup:test-sandbox
+
+  until docker exec $sandbox /bin/sh -c 'docker info' > /dev/null 2>&1; do
+    sleep 0.5
+  done
+  sleep 0.5
+
+  docker exec $sandbox /bin/sh -c "docker load -i /cache/image.tar.gz"
+  docker exec -e TEST_VERSION=$IMAGE_TAG $sandbox /bin/sh -c "/code/test/$test"
+
+  docker rm $(docker stop $sandbox)
+  docker volume rm $sandbox
   echo ""
   echo "$test passed"
   echo ""

test/user/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '2.4'
+
+services:
+  alpine:
+    image: alpine:3.17.3
+    tty: true 
+    volumes:
+      - app_data:/tmp
+    labels:
+      - docker-volume-backup.archive-pre.user=testuser
+      - docker-volume-backup.archive-pre=/bin/sh -c 'whoami > /tmp/whoami.txt'
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      EXEC_FORWARD_OUTPUT: "true"
+    volumes:
+      - ${LOCAL_DIR:-./local}:/archive
+      - app_data:/backup/data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  app_data:
+  archive:

test/user/run.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
+
+echo "LOCAL_DIR $LOCAL_DIR"
+echo "TMP_DIR $TMP_DIR"
+
+docker compose up -d --quiet-pull
+user_name=testuser
+docker exec user-alpine-1 adduser --disabled-password "$user_name"
+
+docker compose exec backup backup
+
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C "$TMP_DIR"
+if [ ! -f "$TMP_DIR/backup/data/whoami.txt" ]; then
+  fail "Could not find file written by pre command."
+fi
+pass "Found expected file."
+
+if [ "$(cat $TMP_DIR/backup/data/whoami.txt)" != "$user_name" ]; then
+  fail "Could not find expected user name."
+fi
+pass "Found expected user."

test/util.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+set -e
+
+info () {
+  echo "[test:${current_test:-none}:info] "$1""
+}
+
+pass () {
+  echo "[test:${current_test:-none}:pass] "$1""
+}
+
+fail () {
+  echo "[test:${current_test:-none}:fail] "$1""
+  exit 1
+}
+
+skip () {
+  echo "[test:${current_test:-none}:skip] "$1""
+  exit 0
+}
+
+expect_running_containers () {
+  if [ "$(docker ps -q | wc -l)" != "$1" ]; then
+    fail "Expected $1 containers to be running, instead seen: "$(docker ps -a | wc -l)""
+  fi
+  pass "$1 containers running."
+}
+
+docker() {
+  case $1 in
+    compose)
+      shift
+      case $1 in
+        up)
+          shift
+          command docker compose up --timeout 3 "$@";;
+        down)
+          shift
+          command docker compose down --timeout 3 "$@";;
+        *)
+          command docker compose "$@";;
+      esac
+      ;;
+    *)
+      command docker "$@";;
+  esac
+}

test/webdav/docker-compose.yml

@@ -0,0 +1,45 @@
+version: '3'
+
+services:
+  webdav:
+    image: bytemark/webdav:2.4
+    environment:
+      AUTH_TYPE: Digest
+      USERNAME: test
+      PASSWORD: test
+    volumes:
+      - webdav_backup_data:/var/lib/dav
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    depends_on:
+      - webdav
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      WEBDAV_URL: http://webdav/
+      WEBDAV_URL_INSECURE: 'true'
+      WEBDAV_PATH: /my/new/path/
+      WEBDAV_USERNAME: test
+      WEBDAV_PASSWORD: test
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  webdav_backup_data:
+    name: webdav_backup_data
+  app_data:

test/webdav/run.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+docker run --rm \
+  -v webdav_backup_data:/webdav_data \
+  alpine \
+  ash -c 'tar -xvf /webdav_data/data/my/new/path/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in untared remote backup."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+docker run --rm \
+  -v webdav_backup_data:/webdav_data \
+  alpine \
+  ash -c '[ $(find /webdav_data/data/my/new/path/ -type f | wc -l) = "1" ]'
+
+pass "Remote backups have not been deleted."
+
+# The third part of this test checks if old backups get deleted when the retention
+# is set to 7 days (which it should)
+
+BACKUP_RETENTION_DAYS="7" docker compose up -d
+sleep 5
+
+info "Create first backup with no prune"
+docker compose exec backup backup
+
+# Set the modification date of the old backup to 14 days ago
+docker run --rm \
+  -v webdav_backup_data:/webdav_data \
+  --user 82 \
+  alpine \
+  ash -c 'touch -d@$(( $(date +%s) - 1209600 )) /webdav_data/data/my/new/path/test-hostnametoken-old.tar.gz'
+
+info "Create second backup and prune"
+docker compose exec backup backup
+
+docker run --rm \
+  -v webdav_backup_data:/webdav_data \
+  alpine \
+  ash -c 'test ! -f /webdav_data/data/my/new/path/test-hostnametoken-old.tar.gz && test -f /webdav_data/data/my/new/path/test-hostnametoken.tar.gz'
+
+pass "Old remote backup has been pruned, new one is still present."

test/zstd/run.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker network create test_network
+docker volume create app_data
+
+LOCAL_DIR=$(mktemp -d)
+
+docker run -d -q \
+  --name offen \
+  --network test_network \
+  -v app_data:/var/opt/offen/ \
+  offen/offen:latest
+
+sleep 10
+
+docker run --rm -q \
+  --network test_network \
+  -v app_data:/backup/app_data \
+  -v $LOCAL_DIR:/archive \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  --env BACKUP_COMPRESSION=zst \
+  --env BACKUP_FILENAME='test.{{ .Extension }}' \
+  --entrypoint backup \
+  offen/docker-volume-backup:${TEST_VERSION:-canary}
+
+tmp_dir=$(mktemp -d)
+tar -xvf "$LOCAL_DIR/test.tar.zst" --zstd -C $tmp_dir
+if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then
+  fail "Could not find expected file in untared archive."
+fi
+pass "Found relevant files in untared local backup."
+
+# This test does not stop containers during backup. This is happening on
+# purpose in order to cover this setup as well.
+expect_running_containers "1"