Multi platform build requires explicit buildx setup

* Add user option for docker exec

* Add test for user option

* Return test version for image

* remove gitea config file

* refactor tests

* remove comments & fix image name

* add docs

* cleanup

* Update README.md with suggested correction

Co-authored-by: Frederik Ring <frederik.ring@gmail.com>

* fix backup command & bind folder instead of volume

---------

Co-authored-by: tao <generaltao.md@gmail.com>
Co-authored-by: Frederik Ring <frederik.ring@gmail.com>

.circleci/config.yml

@@ -1,63 +0,0 @@
-version: 2.1
-
-jobs:
-  canary:
-    machine:
-      image: ubuntu-1604:202007-01
-    working_directory: ~/docker-volume-backup
-    steps:
-      - checkout
-      - run:
-          name: Build
-          command: |
-            docker build . -t offen/docker-volume-backup:canary
-      - run:
-          name: Run tests
-          working_directory: ~/docker-volume-backup/test
-          command: |
-            ./test.sh canary
-
-  build:
-    docker:
-      - image: cimg/base:2020.06
-        environment:
-          DOCKER_BUILDKIT: '1'
-          DOCKER_CLI_EXPERIMENTAL: enabled
-    working_directory: ~/docker-volume-backup
-    steps:
-      - checkout
-      - setup_remote_docker:
-          version: 20.10.6
-      - docker/install-docker-credential-helper
-      - docker/configure-docker-credentials-store
-      - run:
-          name: Push to Docker Hub
-          command: |
-            echo "$DOCKER_ACCESSTOKEN" | docker login --username offen --password-stdin
-            # This is required for building ARM: https://gitlab.alpinelinux.org/alpine/aports/-/issues/12406
-            docker run --rm --privileged linuxkit/binfmt:v0.8
-            docker context create docker-volume-backup
-            docker buildx create docker-volume-backup --name docker-volume-backup --use
-            docker buildx inspect --bootstrap
-            docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 \
-              -t offen/docker-volume-backup:$CIRCLE_TAG \
-              -t offen/docker-volume-backup:latest \
-              . --push
-
-workflows:
-  version: 2
-  docker_image:
-    jobs:
-      - canary:
-          filters:
-            tags:
-              ignore: /^v.*/
-      - build:
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: /^v.*/
-
-orbs:
-  docker: circleci/docker@1.0.1

.dockerignore

@@ -1 +1,7 @@
 test
+.github
+.circleci
+docs
+.editorconfig
+LICENSE
+README.md

.editorconfig

@@ -0,0 +1,18 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.go]
+indent_style = tab

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+github: offen
+patreon: offen
+

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+<!--
+A clear and concise description of what the bug is.
+-->
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. ...
+2. ...
+3. ...
+
+**Expected behavior**
+<!--
+A clear and concise description of what you expected to happen.
+-->
+
+**Version (please complete the following information):**
+ - Image Version: <!-- e.g. v2.21.0 -->
+ - Docker Version: <!-- e.g. 20.10.17 -->
+ - Docker Compose Version (if applicable): <!-- e.g. 1.29.2 -->
+
+**Additional context**
+<!--
+Add any other context about the problem here.
+-->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,28 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+<!--
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+-->
+
+**Describe the solution you'd like**
+<!--
+A clear and concise description of what you want to happen.
+-->
+
+**Describe alternatives you've considered**
+<!--
+A clear and concise description of any alternative solutions or features you've considered.
+-->
+
+**Additional context**
+<!--
+Add any other context or screenshots about the feature request here.
+-->

.github/ISSUE_TEMPLATE/support_request.md

@@ -0,0 +1,28 @@
+---
+name: Support request
+about: Ask for help
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**What are you trying to do?**
+<!--
+A clear and concise description of what you are trying to do, but cannot get working.
+-->
+
+**What is your current configuration?**
+<!--
+Add the full configuration you are using. Please redact out any real-world credentials.
+-->
+
+**Log output**
+<!--
+Provide the full log output of your setup.
+-->
+
+**Additional context**
+<!--
+Add any other context or screenshots about the support request here.
+-->

.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: Release Docker Image
+
+on:
+  push:
+    tags: v**
+
+jobs:
+  push_to_registries:
+    name: Push Docker image to multiple registries
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker tags
+        id: meta
+        run: |
+          version_tag="${{github.ref_name}}"
+          tags=($version_tag)
+          if [[ "$version_tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            # prerelease tags like `v2.0.0-alpha.1` should not be released as `latest` nor `v2`
+            tags+=("latest")
+            tags+=($(echo "$version_tag" | cut -d. -f1))
+          fi
+          releases=""
+          for tag in "${tags[@]}"; do
+            releases="${releases:+$releases,}offen/docker-volume-backup:$tag,ghcr.io/offen/docker-volume-backup:$tag"
+          done
+          echo "releases=$releases" >> "$GITHUB_OUTPUT"
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          tags: ${{ steps.meta.outputs.releases }}

.github/workflows/test.yml

@@ -0,0 +1,18 @@
+name: Run Integration Tests
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build Docker Image
+        run: docker build . -t offen/docker-volume-backup:test
+      - name: Run Tests
+        working-directory: ./test
+        run: |
+          export GPG_TTY=$(tty)
+          ./test.sh test

Dockerfile

@@ -1,23 +1,23 @@
 # Copyright 2021 - Offen Authors <hioffen@posteo.de>
 # SPDX-License-Identifier: MPL-2.0
 
-FROM golang:1.17-alpine as builder
-ARG MC_VERSION=RELEASE.2021-06-13T17-48-22Z
-RUN go install -ldflags "-X github.com/minio/mc/cmd.ReleaseTag=$MC_VERSION" github.com/minio/mc@$MC_VERSION
+FROM golang:1.20-alpine as builder
 
-FROM alpine:3.14
+WORKDIR /app
+COPY . .
+RUN go mod download
+WORKDIR /app/cmd/backup
+RUN go build -o backup .
+
+FROM alpine:3.17
 
 WORKDIR /root
 
-RUN apk add --update ca-certificates docker openrc gnupg
-RUN update-ca-certificates
-RUN rc-update add docker boot
+RUN apk add --no-cache ca-certificates
 
-COPY --from=builder /go/bin/mc /usr/bin/mc
-RUN mc --version
+COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
 
-COPY src/backup.sh src/entrypoint.sh /root/
-RUN chmod +x backup.sh && mv backup.sh /usr/bin/backup \
-  && chmod +x entrypoint.sh
+COPY ./entrypoint.sh /root/
+RUN chmod +x entrypoint.sh
 
 ENTRYPOINT ["/root/entrypoint.sh"]

NOTICE

@@ -1,22 +0,0 @@
-Copyright 2021 Offen Authors <hioffen@posteo.de>
-
-This project contains software that is Copyright (c) 2018 Futurice
-Licensed under the Terms of the MIT License:
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/backup/archive.go

@@ -0,0 +1,133 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+// Portions of this file are taken from package `targz`, Copyright (c) 2014 Fredrik Wallgren
+// Licensed under the MIT License: https://github.com/walle/targz/blob/57fe4206da5abf7dd3901b4af3891ec2f08c7b08/LICENSE
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+func createArchive(files []string, inputFilePath, outputFilePath string) error {
+	inputFilePath = stripTrailingSlashes(inputFilePath)
+	inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
+	if err != nil {
+		return fmt.Errorf("createArchive: error transposing given file paths: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(outputFilePath), 0755); err != nil {
+		return fmt.Errorf("createArchive: error creating output file path: %w", err)
+	}
+
+	if err := compress(files, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
+		return fmt.Errorf("createArchive: error creating archive: %w", err)
+	}
+
+	return nil
+}
+
+func stripTrailingSlashes(path string) string {
+	if len(path) > 0 && path[len(path)-1] == '/' {
+		path = path[0 : len(path)-1]
+	}
+
+	return path
+}
+
+func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error) {
+	inputFilePath, err := filepath.Abs(inputFilePath)
+	if err == nil {
+		outputFilePath, err = filepath.Abs(outputFilePath)
+	}
+
+	return inputFilePath, outputFilePath, err
+}
+
+func compress(paths []string, outFilePath, subPath string) error {
+	file, err := os.Create(outFilePath)
+	if err != nil {
+		return fmt.Errorf("compress: error creating out file: %w", err)
+	}
+
+	prefix := path.Dir(outFilePath)
+	gzipWriter := gzip.NewWriter(file)
+	tarWriter := tar.NewWriter(gzipWriter)
+
+	for _, p := range paths {
+		if err := writeTarGz(p, tarWriter, prefix); err != nil {
+			return fmt.Errorf("compress: error writing %s to archive: %w", p, err)
+		}
+	}
+
+	err = tarWriter.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing tar writer: %w", err)
+	}
+
+	err = gzipWriter.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing gzip writer: %w", err)
+	}
+
+	err = file.Close()
+	if err != nil {
+		return fmt.Errorf("compress: error closing file: %w", err)
+	}
+
+	return nil
+}
+
+func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
+	fileInfo, err := os.Lstat(path)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err)
+	}
+
+	if fileInfo.Mode()&os.ModeSocket == os.ModeSocket {
+		return nil
+	}
+
+	var link string
+	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
+		var err error
+		if link, err = os.Readlink(path); err != nil {
+			return fmt.Errorf("writeTarGz: error resolving symlink %s: %w", path, err)
+		}
+	}
+
+	header, err := tar.FileInfoHeader(fileInfo, link)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error getting file info header: %w", err)
+	}
+	header.Name = strings.TrimPrefix(path, prefix)
+
+	err = tarWriter.WriteHeader(header)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error writing file info header: %w", err)
+	}
+
+	if !fileInfo.Mode().IsRegular() {
+		return nil
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error opening %s: %w", path, err)
+	}
+	defer file.Close()
+
+	_, err = io.Copy(tarWriter, file)
+	if err != nil {
+		return fmt.Errorf("writeTarGz: error copying %s to tar writer: %w", path, err)
+	}
+
+	return nil
+}

cmd/backup/config.go

@@ -0,0 +1,119 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"time"
+)
+
+// Config holds all configuration values that are expected to be set
+// by users.
+type Config struct {
+	AwsS3BucketName               string        `split_words:"true"`
+	AwsS3Path                     string        `split_words:"true"`
+	AwsEndpoint                   string        `split_words:"true" default:"s3.amazonaws.com"`
+	AwsEndpointProto              string        `split_words:"true" default:"https"`
+	AwsEndpointInsecure           bool          `split_words:"true"`
+	AwsEndpointCACert             CertDecoder   `envconfig:"AWS_ENDPOINT_CA_CERT"`
+	AwsStorageClass               string        `split_words:"true"`
+	AwsAccessKeyID                string        `envconfig:"AWS_ACCESS_KEY_ID"`
+	AwsAccessKeyIDFile            string        `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
+	AwsSecretAccessKey            string        `split_words:"true"`
+	AwsSecretAccessKeyFile        string        `split_words:"true"`
+	AwsIamRoleEndpoint            string        `split_words:"true"`
+	BackupSources                 string        `split_words:"true" default:"/backup"`
+	BackupFilename                string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
+	BackupFilenameExpand          bool          `split_words:"true"`
+	BackupLatestSymlink           string        `split_words:"true"`
+	BackupArchive                 string        `split_words:"true" default:"/archive"`
+	BackupRetentionDays           int32         `split_words:"true" default:"-1"`
+	BackupPruningLeeway           time.Duration `split_words:"true" default:"1m"`
+	BackupPruningPrefix           string        `split_words:"true"`
+	BackupStopContainerLabel      string        `split_words:"true" default:"true"`
+	BackupFromSnapshot            bool          `split_words:"true"`
+	BackupExcludeRegexp           RegexpDecoder `split_words:"true"`
+	GpgPassphrase                 string        `split_words:"true"`
+	NotificationURLs              []string      `envconfig:"NOTIFICATION_URLS"`
+	NotificationLevel             string        `split_words:"true" default:"error"`
+	EmailNotificationRecipient    string        `split_words:"true"`
+	EmailNotificationSender       string        `split_words:"true" default:"noreply@nohost"`
+	EmailSMTPHost                 string        `envconfig:"EMAIL_SMTP_HOST"`
+	EmailSMTPPort                 int           `envconfig:"EMAIL_SMTP_PORT" default:"587"`
+	EmailSMTPUsername             string        `envconfig:"EMAIL_SMTP_USERNAME"`
+	EmailSMTPPassword             string        `envconfig:"EMAIL_SMTP_PASSWORD"`
+	WebdavUrl                     string        `split_words:"true"`
+	WebdavUrlInsecure             bool          `split_words:"true"`
+	WebdavPath                    string        `split_words:"true" default:"/"`
+	WebdavUsername                string        `split_words:"true"`
+	WebdavPassword                string        `split_words:"true"`
+	SSHHostName                   string        `split_words:"true"`
+	SSHPort                       string        `split_words:"true" default:"22"`
+	SSHUser                       string        `split_words:"true"`
+	SSHPassword                   string        `split_words:"true"`
+	SSHIdentityFile               string        `split_words:"true" default:"/root/.ssh/id_rsa"`
+	SSHIdentityPassphrase         string        `split_words:"true"`
+	SSHRemotePath                 string        `split_words:"true"`
+	ExecLabel                     string        `split_words:"true"`
+	ExecForwardOutput             bool          `split_words:"true"`
+	LockTimeout                   time.Duration `split_words:"true" default:"60m"`
+	AzureStorageAccountName       string        `split_words:"true"`
+	AzureStoragePrimaryAccountKey string        `split_words:"true"`
+	AzureStorageContainerName     string        `split_words:"true"`
+	AzureStoragePath              string        `split_words:"true"`
+	AzureStorageEndpoint          string        `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
+}
+
+func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
+	if secretPath == "" {
+		return envVar, nil
+	}
+	data, err := os.ReadFile(secretPath)
+	if err != nil {
+		return "", fmt.Errorf("resolveSecret: error reading secret path: %w", err)
+	}
+	return string(data), nil
+}
+
+type CertDecoder struct {
+	Cert *x509.Certificate
+}
+
+func (c *CertDecoder) Decode(v string) error {
+	if v == "" {
+		return nil
+	}
+	content, err := ioutil.ReadFile(v)
+	if err != nil {
+		content = []byte(v)
+	}
+	block, _ := pem.Decode(content)
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return fmt.Errorf("config: error parsing certificate: %w", err)
+	}
+	*c = CertDecoder{Cert: cert}
+	return nil
+}
+
+type RegexpDecoder struct {
+	Re *regexp.Regexp
+}
+
+func (r *RegexpDecoder) Decode(v string) error {
+	if v == "" {
+		return nil
+	}
+	re, err := regexp.Compile(v)
+	if err != nil {
+		return fmt.Errorf("config: error compiling given regexp `%s`: %w", v, err)
+	}
+	*r = RegexpDecoder{Re: re}
+	return nil
+}

cmd/backup/exec.go

@@ -0,0 +1,207 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+// Portions of this file are taken and adapted from `moby`, Copyright 2012-2017 Docker, Inc.
+// Licensed under the Apache 2.0 License: https://github.com/moby/moby/blob/8e610b2b55bfd1bfa9436ab110d311f5e8a74dcb/LICENSE
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/cosiner/argv"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/pkg/stdcopy"
+	"golang.org/x/sync/errgroup"
+)
+
+func (s *script) exec(containerRef string, command string, user string) ([]byte, []byte, error) {
+	args, _ := argv.Argv(command, nil, nil)
+	commandEnv := []string{
+		fmt.Sprintf("COMMAND_RUNTIME_ARCHIVE_FILEPATH=%s", s.file),
+	}
+	execID, err := s.cli.ContainerExecCreate(context.Background(), containerRef, types.ExecConfig{
+		Cmd:          args[0],
+		AttachStdin:  true,
+		AttachStderr: true,
+		Env:          commandEnv,
+		User:         user,
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error creating container exec: %w", err)
+	}
+
+	resp, err := s.cli.ContainerExecAttach(context.Background(), execID.ID, types.ExecStartCheck{})
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error attaching container exec: %w", err)
+	}
+	defer resp.Close()
+
+	var outBuf, errBuf bytes.Buffer
+	outputDone := make(chan error)
+
+	go func() {
+		_, err := stdcopy.StdCopy(&outBuf, &errBuf, resp.Reader)
+		outputDone <- err
+	}()
+
+	select {
+	case err := <-outputDone:
+		if err != nil {
+			return nil, nil, fmt.Errorf("exec: error demultiplexing output: %w", err)
+		}
+		break
+	}
+
+	stdout, err := ioutil.ReadAll(&outBuf)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error reading stdout: %w", err)
+	}
+	stderr, err := ioutil.ReadAll(&errBuf)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error reading stderr: %w", err)
+	}
+
+	res, err := s.cli.ContainerExecInspect(context.Background(), execID.ID)
+	if err != nil {
+		return nil, nil, fmt.Errorf("exec: error inspecting container exec: %w", err)
+	}
+
+	if res.ExitCode > 0 {
+		return stdout, stderr, fmt.Errorf("exec: running command exited %d", res.ExitCode)
+	}
+
+	return stdout, stderr, nil
+}
+
+func (s *script) runLabeledCommands(label string) error {
+	f := []filters.KeyValuePair{
+		{Key: "label", Value: label},
+	}
+	if s.c.ExecLabel != "" {
+		f = append(f, filters.KeyValuePair{
+			Key:   "label",
+			Value: fmt.Sprintf("docker-volume-backup.exec-label=%s", s.c.ExecLabel),
+		})
+	}
+	containersWithCommand, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Quiet:   true,
+		Filters: filters.NewArgs(f...),
+	})
+	if err != nil {
+		return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
+	}
+
+	var hasDeprecatedContainers bool
+	if label == "docker-volume-backup.archive-pre" {
+		f[0] = filters.KeyValuePair{
+			Key:   "label",
+			Value: "docker-volume-backup.exec-pre",
+		}
+		deprecatedContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+			Quiet:   true,
+			Filters: filters.NewArgs(f...),
+		})
+		if err != nil {
+			return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
+		}
+		if len(deprecatedContainers) != 0 {
+			hasDeprecatedContainers = true
+			containersWithCommand = append(containersWithCommand, deprecatedContainers...)
+		}
+	}
+
+	if label == "docker-volume-backup.archive-post" {
+		f[0] = filters.KeyValuePair{
+			Key:   "label",
+			Value: "docker-volume-backup.exec-post",
+		}
+		deprecatedContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+			Quiet:   true,
+			Filters: filters.NewArgs(f...),
+		})
+		if err != nil {
+			return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
+		}
+		if len(deprecatedContainers) != 0 {
+			hasDeprecatedContainers = true
+			containersWithCommand = append(containersWithCommand, deprecatedContainers...)
+		}
+	}
+
+	if len(containersWithCommand) == 0 {
+		return nil
+	}
+
+	if hasDeprecatedContainers {
+		s.logger.Warn(
+			"Using `docker-volume-backup.exec-pre` and `docker-volume-backup.exec-post` labels has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use other `-pre` and `-post` labels instead. Refer to the README for an upgrade guide.",
+		)
+	}
+
+	g := new(errgroup.Group)
+
+	for _, container := range containersWithCommand {
+		c := container
+		g.Go(func() error {
+			cmd, ok := c.Labels[label]
+			if !ok && label == "docker-volume-backup.archive-pre" {
+				cmd, _ = c.Labels["docker-volume-backup.exec-pre"]
+			} else if !ok && label == "docker-volume-backup.archive-post" {
+				cmd, _ = c.Labels["docker-volume-backup.exec-post"]
+			}
+
+			userLabelName := fmt.Sprintf("%s.user", label)
+			user := c.Labels[userLabelName]
+
+			s.logger.Infof("Running %s command %s for container %s", label, cmd, strings.TrimPrefix(c.Names[0], "/"))
+			stdout, stderr, err := s.exec(c.ID, cmd, user)
+			if s.c.ExecForwardOutput {
+				os.Stderr.Write(stderr)
+				os.Stdout.Write(stdout)
+			}
+			if err != nil {
+				return fmt.Errorf("runLabeledCommands: error executing command: %w", err)
+			}
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		return fmt.Errorf("runLabeledCommands: error from errgroup: %w", err)
+	}
+	return nil
+}
+
+type lifecyclePhase string
+
+const (
+	lifecyclePhaseArchive lifecyclePhase = "archive"
+	lifecyclePhaseProcess lifecyclePhase = "process"
+	lifecyclePhaseCopy    lifecyclePhase = "copy"
+	lifecyclePhasePrune   lifecyclePhase = "prune"
+)
+
+func (s *script) withLabeledCommands(step lifecyclePhase, cb func() error) func() error {
+	if s.cli == nil {
+		return cb
+	}
+	return func() error {
+		if err := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil {
+			return fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err)
+		}
+		defer func() {
+			s.must(s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step)))
+		}()
+		return cb()
+	}
+}

cmd/backup/hooks.go

@@ -0,0 +1,57 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+)
+
+// hook contains a queued action that can be trigger them when the script
+// reaches a certain point (e.g. unsuccessful backup)
+type hook struct {
+	level  hookLevel
+	action func(err error) error
+}
+
+type hookLevel int
+
+const (
+	hookLevelPlumbing hookLevel = iota
+	hookLevelError
+	hookLevelInfo
+)
+
+var hookLevels = map[string]hookLevel{
+	"info":  hookLevelInfo,
+	"error": hookLevelError,
+}
+
+// registerHook adds the given action at the given level.
+func (s *script) registerHook(level hookLevel, action func(err error) error) {
+	s.hooks = append(s.hooks, hook{level, action})
+}
+
+// runHooks runs all hooks that have been registered using the
+// given levels in the defined ordering. In case executing a hook returns an
+// error, the following hooks will still be run before the function returns.
+func (s *script) runHooks(err error) error {
+	sort.SliceStable(s.hooks, func(i, j int) bool {
+		return s.hooks[i].level < s.hooks[j].level
+	})
+	var actionErrors []error
+	for _, hook := range s.hooks {
+		if hook.level > s.hookLevel {
+			continue
+		}
+		if actionErr := hook.action(err); actionErr != nil {
+			actionErrors = append(actionErrors, fmt.Errorf("runHooks: error running hook: %w", actionErr))
+		}
+	}
+	if len(actionErrors) != 0 {
+		return errors.Join(actionErrors...)
+	}
+	return nil
+}

cmd/backup/lock.go

@@ -0,0 +1,58 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/gofrs/flock"
+)
+
+// lock opens a lockfile at the given location, keeping it locked until the
+// caller invokes the returned release func. In case the lock is currently blocked
+// by another execution, it will repeatedly retry until the lock is available
+// or the given timeout is exceeded.
+func (s *script) lock(lockfile string) (func() error, error) {
+	start := time.Now()
+	defer func() {
+		s.stats.LockedTime = time.Now().Sub(start)
+	}()
+
+	retry := time.NewTicker(5 * time.Second)
+	defer retry.Stop()
+	deadline := time.NewTimer(s.c.LockTimeout)
+	defer deadline.Stop()
+
+	fileLock := flock.New(lockfile)
+
+	for {
+		acquired, err := fileLock.TryLock()
+		if err != nil {
+			return noop, fmt.Errorf("lock: error trying to lock: %w", err)
+		}
+		if acquired {
+			if s.encounteredLock {
+				s.logger.Info("Acquired exclusive lock on subsequent attempt, ready to continue.")
+			}
+			return fileLock.Unlock, nil
+		}
+
+		if !s.encounteredLock {
+			s.logger.Infof(
+				"Exclusive lock was not available on first attempt. Will retry until it becomes available or the timeout of %s is exceeded.",
+				s.c.LockTimeout,
+			)
+			s.encounteredLock = true
+		}
+
+		select {
+		case <-retry.C:
+			continue
+		case <-deadline.C:
+			return noop, errors.New("lock: timed out waiting for lockfile to become available")
+		}
+	}
+}

cmd/backup/main.go

@@ -0,0 +1,58 @@
+// Copyright 2021-2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"os"
+)
+
+func main() {
+	s, err := newScript()
+	if err != nil {
+		panic(err)
+	}
+
+	unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
+	defer unlock()
+	s.must(err)
+
+	defer func() {
+		if pArg := recover(); pArg != nil {
+			if err, ok := pArg.(error); ok {
+				if hookErr := s.runHooks(err); hookErr != nil {
+					s.logger.Errorf("An error occurred calling the registered hooks: %s", hookErr)
+				}
+				os.Exit(1)
+			}
+			panic(pArg)
+		}
+
+		if err := s.runHooks(nil); err != nil {
+			s.logger.Errorf(
+				"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
+				err,
+			)
+			os.Exit(1)
+		}
+		s.logger.Info("Finished running backup tasks.")
+	}()
+
+	s.must(s.withLabeledCommands(lifecyclePhaseArchive, func() error {
+		restartContainers, err := s.stopContainers()
+		// The mechanism for restarting containers is not using hooks as it
+		// should happen as soon as possible (i.e. before uploading backups or
+		// similar).
+		defer func() {
+			s.must(restartContainers())
+		}()
+		if err != nil {
+			return err
+		}
+		return s.createArchive()
+	})())
+
+	s.must(s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)())
+	s.must(s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)())
+	s.must(s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)())
+}

cmd/backup/notifications.go

@@ -0,0 +1,108 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"fmt"
+	"os"
+	"text/template"
+	"time"
+
+	sTypes "github.com/containrrr/shoutrrr/pkg/types"
+)
+
+//go:embed notifications.tmpl
+var defaultNotifications string
+
+// NotificationData data to be passed to the notification templates
+type NotificationData struct {
+	Error  error
+	Config *Config
+	Stats  *Stats
+}
+
+// notify sends a notification using the given title and body templates.
+// Automatically creates notification data, adding the given error
+func (s *script) notify(titleTemplate string, bodyTemplate string, err error) error {
+	params := NotificationData{
+		Error:  err,
+		Stats:  s.stats,
+		Config: s.c,
+	}
+
+	titleBuf := &bytes.Buffer{}
+	if err := s.template.ExecuteTemplate(titleBuf, titleTemplate, params); err != nil {
+		return fmt.Errorf("notify: error executing %s template: %w", titleTemplate, err)
+	}
+
+	bodyBuf := &bytes.Buffer{}
+	if err := s.template.ExecuteTemplate(bodyBuf, bodyTemplate, params); err != nil {
+		return fmt.Errorf("notify: error executing %s template: %w", bodyTemplate, err)
+	}
+
+	if err := s.sendNotification(titleBuf.String(), bodyBuf.String()); err != nil {
+		return fmt.Errorf("notify: error notifying: %w", err)
+	}
+	return nil
+}
+
+// notifyFailure sends a notification about a failed backup run
+func (s *script) notifyFailure(err error) error {
+	return s.notify("title_failure", "body_failure", err)
+}
+
+// notifyFailure sends a notification about a successful backup run
+func (s *script) notifySuccess() error {
+	return s.notify("title_success", "body_success", nil)
+}
+
+// sendNotification sends a notification to all configured third party services
+func (s *script) sendNotification(title, body string) error {
+	var errs []error
+	for _, result := range s.sender.Send(body, &sTypes.Params{"title": title}) {
+		if result != nil {
+			errs = append(errs, result)
+		}
+	}
+	if len(errs) != 0 {
+		return fmt.Errorf("sendNotification: error sending message: %w", errors.Join(errs...))
+	}
+	return nil
+}
+
+var templateHelpers = template.FuncMap{
+	"formatTime": func(t time.Time) string {
+		return t.Format(time.RFC3339)
+	},
+	"formatBytesDec": func(bytes uint64) string {
+		return formatBytes(bytes, true)
+	},
+	"formatBytesBin": func(bytes uint64) string {
+		return formatBytes(bytes, false)
+	},
+	"env": os.Getenv,
+}
+
+// formatBytes converts an amount of bytes in a human-readable representation
+// the decimal parameter specifies if using powers of 1000 (decimal) or powers of 1024 (binary)
+func formatBytes(b uint64, decimal bool) string {
+	unit := uint64(1024)
+	format := "%.1f %ciB"
+	if decimal {
+		unit = uint64(1000)
+		format = "%.1f %cB"
+	}
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := unit, 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf(format, float64(b)/float64(div), "kMGTPE"[exp])
+}

cmd/backup/notifications.tmpl

@@ -0,0 +1,26 @@
+{{ define "title_failure" -}}
+Failure running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
+{{- end }}
+
+
+{{ define "body_failure" -}}
+Running docker-volume-backup failed with error: {{ .Error }}
+
+Log output of the failed run was:
+
+{{ .Stats.LogOutput }}
+{{- end }}
+
+
+{{ define "title_success" -}}
+Success running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
+{{- end }}
+
+
+{{ define "body_success" -}}
+Running docker-volume-backup succeeded.
+
+Log output was:
+
+{{ .Stats.LogOutput }}
+{{- end }}

cmd/backup/script.go

@@ -0,0 +1,579 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"text/template"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/offen/docker-volume-backup/internal/storage/azure"
+	"github.com/offen/docker-volume-backup/internal/storage/local"
+	"github.com/offen/docker-volume-backup/internal/storage/s3"
+	"github.com/offen/docker-volume-backup/internal/storage/ssh"
+	"github.com/offen/docker-volume-backup/internal/storage/webdav"
+
+	"github.com/containrrr/shoutrrr"
+	"github.com/containrrr/shoutrrr/pkg/router"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/leekchan/timeutil"
+	"github.com/otiai10/copy"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/crypto/openpgp"
+	"golang.org/x/sync/errgroup"
+)
+
+// script holds all the stateful information required to orchestrate a
+// single backup run.
+type script struct {
+	cli       *client.Client
+	storages  []storage.Backend
+	logger    *logrus.Logger
+	sender    *router.ServiceRouter
+	template  *template.Template
+	hooks     []hook
+	hookLevel hookLevel
+
+	file  string
+	stats *Stats
+
+	encounteredLock bool
+
+	c *Config
+}
+
+// newScript creates all resources needed for the script to perform actions against
+// remote resources like the Docker engine or remote storage locations. All
+// reading from env vars or other configuration sources is expected to happen
+// in this method.
+func newScript() (*script, error) {
+	stdOut, logBuffer := buffer(os.Stdout)
+	s := &script{
+		c: &Config{},
+		logger: &logrus.Logger{
+			Out:       stdOut,
+			Formatter: new(logrus.TextFormatter),
+			Hooks:     make(logrus.LevelHooks),
+			Level:     logrus.InfoLevel,
+		},
+		stats: &Stats{
+			StartTime: time.Now(),
+			LogOutput: logBuffer,
+			Storages: map[string]StorageStats{
+				"S3":     {},
+				"WebDAV": {},
+				"SSH":    {},
+				"Local":  {},
+				"Azure":  {},
+			},
+		},
+	}
+
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		s.stats.EndTime = time.Now()
+		s.stats.TookTime = s.stats.EndTime.Sub(s.stats.StartTime)
+		return nil
+	})
+
+	if err := envconfig.Process("", s.c); err != nil {
+		return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
+	}
+
+	s.file = path.Join("/tmp", s.c.BackupFilename)
+	if s.c.BackupFilenameExpand {
+		s.file = os.ExpandEnv(s.file)
+		s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
+		s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix)
+	}
+	s.file = timeutil.Strftime(&s.stats.StartTime, s.file)
+
+	_, err := os.Stat("/var/run/docker.sock")
+	_, dockerHostSet := os.LookupEnv("DOCKER_HOST")
+	if !os.IsNotExist(err) || dockerHostSet {
+		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+		if err != nil {
+			return nil, fmt.Errorf("newScript: failed to create docker client")
+		}
+		s.cli = cli
+	}
+
+	logFunc := func(logType storage.LogLevel, context string, msg string, params ...any) {
+		switch logType {
+		case storage.LogLevelWarning:
+			s.logger.Warnf("["+context+"] "+msg, params...)
+		case storage.LogLevelError:
+			s.logger.Errorf("["+context+"] "+msg, params...)
+		case storage.LogLevelInfo:
+		default:
+			s.logger.Infof("["+context+"] "+msg, params...)
+		}
+	}
+
+	if s.c.AwsS3BucketName != "" {
+		accessKeyID, err := s.c.resolveSecret(s.c.AwsAccessKeyID, s.c.AwsAccessKeyIDFile)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: error resolving AwsAccessKeyID: %w", err)
+		}
+		secretAccessKey, err := s.c.resolveSecret(s.c.AwsSecretAccessKey, s.c.AwsSecretAccessKeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: error resolving AwsSecretAccessKey: %w", err)
+		}
+		s3Config := s3.Config{
+			Endpoint:         s.c.AwsEndpoint,
+			AccessKeyID:      accessKeyID,
+			SecretAccessKey:  secretAccessKey,
+			IamRoleEndpoint:  s.c.AwsIamRoleEndpoint,
+			EndpointProto:    s.c.AwsEndpointProto,
+			EndpointInsecure: s.c.AwsEndpointInsecure,
+			RemotePath:       s.c.AwsS3Path,
+			BucketName:       s.c.AwsS3BucketName,
+			StorageClass:     s.c.AwsStorageClass,
+			CACert:           s.c.AwsEndpointCACert.Cert,
+		}
+		if s3Backend, err := s3.NewStorageBackend(s3Config, logFunc); err != nil {
+			return nil, err
+		} else {
+			s.storages = append(s.storages, s3Backend)
+		}
+	}
+
+	if s.c.WebdavUrl != "" {
+		webDavConfig := webdav.Config{
+			URL:         s.c.WebdavUrl,
+			URLInsecure: s.c.WebdavUrlInsecure,
+			Username:    s.c.WebdavUsername,
+			Password:    s.c.WebdavPassword,
+			RemotePath:  s.c.WebdavPath,
+		}
+		if webdavBackend, err := webdav.NewStorageBackend(webDavConfig, logFunc); err != nil {
+			return nil, err
+		} else {
+			s.storages = append(s.storages, webdavBackend)
+		}
+	}
+
+	if s.c.SSHHostName != "" {
+		sshConfig := ssh.Config{
+			HostName:           s.c.SSHHostName,
+			Port:               s.c.SSHPort,
+			User:               s.c.SSHUser,
+			Password:           s.c.SSHPassword,
+			IdentityFile:       s.c.SSHIdentityFile,
+			IdentityPassphrase: s.c.SSHIdentityPassphrase,
+			RemotePath:         s.c.SSHRemotePath,
+		}
+		if sshBackend, err := ssh.NewStorageBackend(sshConfig, logFunc); err != nil {
+			return nil, err
+		} else {
+			s.storages = append(s.storages, sshBackend)
+		}
+	}
+
+	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
+		localConfig := local.Config{
+			ArchivePath:   s.c.BackupArchive,
+			LatestSymlink: s.c.BackupLatestSymlink,
+		}
+		localBackend := local.NewStorageBackend(localConfig, logFunc)
+		s.storages = append(s.storages, localBackend)
+	}
+
+	if s.c.AzureStorageAccountName != "" {
+		azureConfig := azure.Config{
+			ContainerName:     s.c.AzureStorageContainerName,
+			AccountName:       s.c.AzureStorageAccountName,
+			PrimaryAccountKey: s.c.AzureStoragePrimaryAccountKey,
+			Endpoint:          s.c.AzureStorageEndpoint,
+			RemotePath:        s.c.AzureStoragePath,
+		}
+		azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
+		if err != nil {
+			return nil, err
+		}
+		s.storages = append(s.storages, azureBackend)
+	}
+
+	if s.c.EmailNotificationRecipient != "" {
+		emailURL := fmt.Sprintf(
+			"smtp://%s:%s@%s:%d/?from=%s&to=%s",
+			s.c.EmailSMTPUsername,
+			s.c.EmailSMTPPassword,
+			s.c.EmailSMTPHost,
+			s.c.EmailSMTPPort,
+			s.c.EmailNotificationSender,
+			s.c.EmailNotificationRecipient,
+		)
+		s.c.NotificationURLs = append(s.c.NotificationURLs, emailURL)
+		s.logger.Warn(
+			"Using EMAIL_* keys for providing notification configuration has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use NOTIFICATION_URLS instead. Refer to the README for an upgrade guide.",
+		)
+	}
+
+	hookLevel, ok := hookLevels[s.c.NotificationLevel]
+	if !ok {
+		return nil, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel)
+	}
+	s.hookLevel = hookLevel
+
+	if len(s.c.NotificationURLs) > 0 {
+		sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...)
+		if senderErr != nil {
+			return nil, fmt.Errorf("newScript: error creating sender: %w", senderErr)
+		}
+		s.sender = sender
+
+		tmpl := template.New("")
+		tmpl.Funcs(templateHelpers)
+		tmpl, err = tmpl.Parse(defaultNotifications)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
+		}
+
+		if fi, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil && fi.IsDir() {
+			tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*")
+			if err != nil {
+				return nil, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
+			}
+		}
+		s.template = tmpl
+
+		// To prevent duplicate notifications, ensure the regsistered callbacks
+		// run mutually exclusive.
+		s.registerHook(hookLevelError, func(err error) error {
+			if err == nil {
+				return nil
+			}
+			return s.notifyFailure(err)
+		})
+		s.registerHook(hookLevelInfo, func(err error) error {
+			if err != nil {
+				return nil
+			}
+			return s.notifySuccess()
+		})
+	}
+
+	return s, nil
+}
+
+// stopContainers stops all Docker containers that are marked as to being
+// stopped during the backup and returns a function that can be called to
+// restart everything that has been stopped.
+func (s *script) stopContainers() (func() error, error) {
+	if s.cli == nil {
+		return noop, nil
+	}
+
+	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Quiet: true,
+	})
+	if err != nil {
+		return noop, fmt.Errorf("stopContainers: error querying for containers: %w", err)
+	}
+
+	containerLabel := fmt.Sprintf(
+		"docker-volume-backup.stop-during-backup=%s",
+		s.c.BackupStopContainerLabel,
+	)
+	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Quiet: true,
+		Filters: filters.NewArgs(filters.KeyValuePair{
+			Key:   "label",
+			Value: containerLabel,
+		}),
+	})
+
+	if err != nil {
+		return noop, fmt.Errorf("stopContainers: error querying for containers to stop: %w", err)
+	}
+
+	if len(containersToStop) == 0 {
+		return noop, nil
+	}
+
+	s.logger.Infof(
+		"Stopping %d container(s) labeled `%s` out of %d running container(s).",
+		len(containersToStop),
+		containerLabel,
+		len(allContainers),
+	)
+
+	var stoppedContainers []types.Container
+	var stopErrors []error
+	for _, container := range containersToStop {
+		if err := s.cli.ContainerStop(context.Background(), container.ID, nil); err != nil {
+			stopErrors = append(stopErrors, err)
+		} else {
+			stoppedContainers = append(stoppedContainers, container)
+		}
+	}
+
+	var stopError error
+	if len(stopErrors) != 0 {
+		stopError = fmt.Errorf(
+			"stopContainers: %d error(s) stopping containers: %w",
+			len(stopErrors),
+			errors.Join(stopErrors...),
+		)
+	}
+
+	s.stats.Containers = ContainersStats{
+		All:     uint(len(allContainers)),
+		ToStop:  uint(len(containersToStop)),
+		Stopped: uint(len(stoppedContainers)),
+	}
+
+	return func() error {
+		servicesRequiringUpdate := map[string]struct{}{}
+
+		var restartErrors []error
+		for _, container := range stoppedContainers {
+			if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok {
+				servicesRequiringUpdate[swarmServiceName] = struct{}{}
+				continue
+			}
+			if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil {
+				restartErrors = append(restartErrors, err)
+			}
+		}
+
+		if len(servicesRequiringUpdate) != 0 {
+			services, _ := s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
+			for serviceName := range servicesRequiringUpdate {
+				var serviceMatch swarm.Service
+				for _, service := range services {
+					if service.Spec.Name == serviceName {
+						serviceMatch = service
+						break
+					}
+				}
+				if serviceMatch.ID == "" {
+					return fmt.Errorf("stopContainers: couldn't find service with name %s", serviceName)
+				}
+				serviceMatch.Spec.TaskTemplate.ForceUpdate = 1
+				if _, err := s.cli.ServiceUpdate(
+					context.Background(), serviceMatch.ID,
+					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},
+				); err != nil {
+					restartErrors = append(restartErrors, err)
+				}
+			}
+		}
+
+		if len(restartErrors) != 0 {
+			return fmt.Errorf(
+				"stopContainers: %d error(s) restarting containers and services: %w",
+				len(restartErrors),
+				errors.Join(restartErrors...),
+			)
+		}
+		s.logger.Infof(
+			"Restarted %d container(s) and the matching service(s).",
+			len(stoppedContainers),
+		)
+		return nil
+	}, stopError
+}
+
+// createArchive creates a tar archive of the configured backup location and
+// saves it to disk.
+func (s *script) createArchive() error {
+	backupSources := s.c.BackupSources
+
+	if s.c.BackupFromSnapshot {
+		s.logger.Warn(
+			"Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the README for an upgrade guide.",
+		)
+		backupSources = filepath.Join("/tmp", s.c.BackupSources)
+		// copy before compressing guard against a situation where backup folder's content are still growing.
+		s.registerHook(hookLevelPlumbing, func(error) error {
+			if err := remove(backupSources); err != nil {
+				return fmt.Errorf("createArchive: error removing snapshot: %w", err)
+			}
+			s.logger.Infof("Removed snapshot `%s`.", backupSources)
+			return nil
+		})
+		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
+			PreserveTimes: true,
+			PreserveOwner: true,
+		}); err != nil {
+			return fmt.Errorf("createArchive: error creating snapshot: %w", err)
+		}
+		s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources)
+	}
+
+	tarFile := s.file
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		if err := remove(tarFile); err != nil {
+			return fmt.Errorf("createArchive: error removing tar file: %w", err)
+		}
+		s.logger.Infof("Removed tar file `%s`.", tarFile)
+		return nil
+	})
+
+	backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources))
+	if err != nil {
+		return fmt.Errorf("createArchive: error getting absolute path: %w", err)
+	}
+
+	var filesEligibleForBackup []string
+	if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) {
+			return nil
+		}
+		filesEligibleForBackup = append(filesEligibleForBackup, path)
+		return nil
+	}); err != nil {
+		return fmt.Errorf("createArchive: error walking filesystem tree: %w", err)
+	}
+
+	if err := createArchive(filesEligibleForBackup, backupSources, tarFile); err != nil {
+		return fmt.Errorf("createArchive: error compressing backup folder: %w", err)
+	}
+
+	s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, tarFile)
+	return nil
+}
+
+// encryptArchive encrypts the backup file using PGP and the configured passphrase.
+// In case no passphrase is given it returns early, leaving the backup file
+// untouched.
+func (s *script) encryptArchive() error {
+	if s.c.GpgPassphrase == "" {
+		return nil
+	}
+
+	gpgFile := fmt.Sprintf("%s.gpg", s.file)
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		if err := remove(gpgFile); err != nil {
+			return fmt.Errorf("encryptArchive: error removing gpg file: %w", err)
+		}
+		s.logger.Infof("Removed GPG file `%s`.", gpgFile)
+		return nil
+	})
+
+	outFile, err := os.Create(gpgFile)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error opening out file: %w", err)
+	}
+	defer outFile.Close()
+
+	_, name := path.Split(s.file)
+	dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
+		IsBinary: true,
+		FileName: name,
+	}, nil)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error encrypting backup file: %w", err)
+	}
+	defer dst.Close()
+
+	src, err := os.Open(s.file)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error opening backup file `%s`: %w", s.file, err)
+	}
+
+	if _, err := io.Copy(dst, src); err != nil {
+		return fmt.Errorf("encryptArchive: error writing ciphertext to file: %w", err)
+	}
+
+	s.file = gpgFile
+	s.logger.Infof("Encrypted backup using given passphrase, saving as `%s`.", s.file)
+	return nil
+}
+
+// copyArchive makes sure the backup file is copied to both local and remote locations
+// as per the given configuration.
+func (s *script) copyArchive() error {
+	_, name := path.Split(s.file)
+	if stat, err := os.Stat(s.file); err != nil {
+		return fmt.Errorf("copyArchive: unable to stat backup file: %w", err)
+	} else {
+		size := stat.Size()
+		s.stats.BackupFile = BackupFileStats{
+			Size:     uint64(size),
+			Name:     name,
+			FullPath: s.file,
+		}
+	}
+
+	eg := errgroup.Group{}
+	for _, backend := range s.storages {
+		b := backend
+		eg.Go(func() error {
+			return b.Copy(s.file)
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return fmt.Errorf("copyArchive: error copying archive: %w", err)
+	}
+
+	return nil
+}
+
+// pruneBackups rotates away backups from local and remote storages using
+// the given configuration. In case the given configuration would delete all
+// backups, it does nothing instead and logs a warning.
+func (s *script) pruneBackups() error {
+	if s.c.BackupRetentionDays < 0 {
+		return nil
+	}
+
+	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
+
+	eg := errgroup.Group{}
+	for _, backend := range s.storages {
+		b := backend
+		eg.Go(func() error {
+			stats, err := b.Prune(deadline, s.c.BackupPruningPrefix)
+			if err != nil {
+				return err
+			}
+			s.stats.Lock()
+			s.stats.Storages[b.Name()] = StorageStats{
+				Total:  stats.Total,
+				Pruned: stats.Pruned,
+			}
+			s.stats.Unlock()
+			return nil
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return fmt.Errorf("pruneBackups: error pruning backups: %w", err)
+	}
+
+	return nil
+}
+
+// must exits the script run prematurely in case the given error
+// is non-nil.
+func (s *script) must(err error) {
+	if err != nil {
+		s.logger.Errorf("Fatal error running backup: %s", err)
+		panic(err)
+	}
+}

cmd/backup/stats.go

@@ -0,0 +1,45 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	"sync"
+	"time"
+)
+
+// ContainersStats stats about the docker containers
+type ContainersStats struct {
+	All        uint
+	ToStop     uint
+	Stopped    uint
+	StopErrors uint
+}
+
+// BackupFileStats stats about the created backup file
+type BackupFileStats struct {
+	Name     string
+	FullPath string
+	Size     uint64
+}
+
+// StorageStats stats about the status of an archival directory
+type StorageStats struct {
+	Total       uint
+	Pruned      uint
+	PruneErrors uint
+}
+
+// Stats global stats regarding script execution
+type Stats struct {
+	sync.Mutex
+	StartTime  time.Time
+	EndTime    time.Time
+	TookTime   time.Duration
+	LockedTime time.Duration
+	LogOutput  *bytes.Buffer
+	Containers ContainersStats
+	BackupFile BackupFileStats
+	Storages   map[string]StorageStats
+}

cmd/backup/util.go

@@ -0,0 +1,52 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+)
+
+var noop = func() error { return nil }
+
+// remove removes the given file or directory from disk.
+func remove(location string) error {
+	fi, err := os.Lstat(location)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("remove: error checking for existence of `%s`: %w", location, err)
+	}
+	if fi.IsDir() {
+		err = os.RemoveAll(location)
+	} else {
+		err = os.Remove(location)
+	}
+	if err != nil {
+		return fmt.Errorf("remove: error removing `%s`: %w", location, err)
+	}
+	return nil
+}
+
+// buffer takes an io.Writer and returns a wrapped version of the
+// writer that writes to both the original target as well as the returned buffer
+func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {
+	buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w}
+	return buffering, &buffering.buf
+}
+
+type bufferingWriter struct {
+	buf    bytes.Buffer
+	writer io.Writer
+}
+
+func (b *bufferingWriter) Write(p []byte) (n int, err error) {
+	if n, err := b.buf.Write(p); err != nil {
+		return n, fmt.Errorf("(*bufferingWriter).Write: error writing to buffer: %w", err)
+	}
+	return b.writer.Write(p)
+}

docs/NOTIFICATION-TEMPLATES.md

@@ -0,0 +1,40 @@
+# Notification templates reference
+
+In order to customize title and body of notifications you'll have to write a [go template](https://pkg.go.dev/text/template) and mount it inside the `/etc/dockervolumebackup/notifications.d/` directory.
+
+Configuration, data about the backup run and helper functions will be passed to this template, this page documents them fully.
+
+## Data
+Here is a list of all data passed to the template:
+
+* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`)
+* `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates
+* `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available.
+  * `StartTime`: time when the script started execution
+  * `EndTime`: time when the backup has completed successfully (after pruning)
+  * `TookTime`: amount of time it took for the backup to run. (equal to `EndTime - StartTime`)
+  * `LockedTime`: amount of time it took for the backup to acquire the exclusive lock
+  * `LogOutput`: full log of the application
+  * `Containers`: object containing stats about the docker containers
+    * `All`: total number of containers
+    * `ToStop`: number of containers matched by the stop rule
+    * `Stopped`: number of containers successfully stopped
+    * `StopErrors`: number of containers that were unable to be stopped (equal to `ToStop - Stopped`)
+  * `BackupFile`: object containing information about the backup file
+    * `Name`: name of the backup file (e.g. `backup-2022-02-11T01-00-00.tar.gz`)
+    * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
+    * `Size`: size in bytes of the backup file
+  * `Storages`: object that holds stats about each storage
+    * `Local`, `S3`, `WebDAV`, `Azure` or `SSH`:
+      * `Total`: total number of backup files
+      * `Pruned`: number of backup files that were deleted due to pruning rule
+      * `PruneErrors`: number of backup files that were unable to be pruned
+
+## Functions
+
+Some formatting and helper functions are also available:
+
+* `formatTime`: formats a time object using [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format (e.g. `2022-02-11T01:00:00Z`)
+* `formatBytesBin`: formats an amount of bytes using powers of 1024 (e.g. `7055258` bytes will be `6.7 MiB`) 
+* `formatBytesDec`: formats an amount of bytes using powers of 1000 (e.g. `7055258` bytes will be `7.1 MB`)
+* `env`: returns the value of the environment variable of the given key if set

entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# Copyright 2021 - Offen Authors <hioffen@posteo.de>
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+if [ ! -d "/etc/dockervolumebackup/conf.d" ]; then
+  BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+
+  echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
+  echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
+else
+  echo "/etc/dockervolumebackup/conf.d was found, using configuration files from this directory."
+
+  crontab -r && crontab /dev/null
+  for file in /etc/dockervolumebackup/conf.d/*; do
+    source $file
+    BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+    echo "Appending cron.d entry with expression $BACKUP_CRON_EXPRESSION and configuration file $file"
+    (crontab -l; echo "$BACKUP_CRON_EXPRESSION /bin/sh -c 'set -a; source $file; set +a && backup' 2>&1") | crontab -
+  done
+fi
+
+echo "Starting cron in foreground."
+crond -f -d 8

go.mod

@@ -0,0 +1,68 @@
+module github.com/offen/docker-volume-backup
+
+go 1.19
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1
+	github.com/containrrr/shoutrrr v0.6.0
+	github.com/cosiner/argv v0.1.0
+	github.com/docker/docker v20.10.11+incompatible
+	github.com/gofrs/flock v0.8.1
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
+	github.com/minio/minio-go/v7 v7.0.44
+	github.com/otiai10/copy v1.7.0
+	github.com/pkg/sftp v1.13.5
+	github.com/sirupsen/logrus v1.9.0
+	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
+	golang.org/x/crypto v0.3.0
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
+)
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/containerd/containerd v1.6.18 // indirect
+	github.com/docker/distribution v2.8.0+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/mux v1.7.3 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.15.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.1 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/moby/term v0.0.0-20200312100748-672ec06f55cd // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/onsi/gomega v1.10.3 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rs/xid v1.4.0 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect
+	google.golang.org/grpc v1.47.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+)

go.sum

@@ -0,0 +1,442 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns=
+github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw=
+github.com/containrrr/shoutrrr v0.6.0 h1:74cijCwvx+xVKd5jMH2uPO3hRW+pOiKwl0JyDp4m8gU=
+github.com/containrrr/shoutrrr v0.6.0/go.mod h1:ye9jGX5YzMnJ76waaNVWlJ4luhMEyt1EWU5unYTQSb0=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg=
+github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8=
+github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c=
+github.com/docker/distribution v2.8.0+incompatible h1:l9EaZDICImO1ngI+uTifW+ZYvvz7fKISBAKpg+MbWbY=
+github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo=
+github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
+github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
+github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
+github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.1 h1:U33DW0aiEj633gHYw3LoDNfkDiYnE5Q8M/TKJn2f2jI=
+github.com/klauspost/cpuid/v2 v2.2.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs=
+github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.44 h1:9zUJ7iU7ax2P1jOvTp6nVrgzlZq3AZlFm0XfRFDKstM=
+github.com/minio/minio-go/v7 v7.0.44/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI=
+github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.6 h1:11TGpSHY7Esh/i/qnq02Jo5oVrI1Gue8Slbq0ujPZFQ=
+github.com/nxadm/tail v1.4.6/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
+github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA=
+github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec=
+github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
+github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI=
+github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
+github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9HY7giGM+kYcnQ71m14JnGdQabMPmyt++8=
+github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
+golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 h1:qRu95HZ148xXw+XeZ3dvqe85PxH4X8+jIo0iRPKcEnM=
+google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8=
+google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/storage/azure/azure.go

@@ -0,0 +1,160 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package azure
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"text/template"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/offen/docker-volume-backup/internal/storage"
+)
+
+type azureBlobStorage struct {
+	*storage.StorageBackend
+	client        *azblob.Client
+	containerName string
+}
+
+// Config contains values that define the configuration of an Azure Blob Storage.
+type Config struct {
+	AccountName       string
+	ContainerName     string
+	PrimaryAccountKey string
+	Endpoint          string
+	RemotePath        string
+}
+
+// NewStorageBackend creates and initializes a new Azure Blob Storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error parsing endpoint template: %w", err)
+	}
+	var ep bytes.Buffer
+	if err := endpointTemplate.Execute(&ep, opts); err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err)
+	}
+	normalizedEndpoint := fmt.Sprintf("%s/", strings.TrimSuffix(ep.String(), "/"))
+
+	var client *azblob.Client
+	if opts.PrimaryAccountKey != "" {
+		cred, err := azblob.NewSharedKeyCredential(opts.AccountName, opts.PrimaryAccountKey)
+		if err != nil {
+			return nil, fmt.Errorf("NewStorageBackend: error creating shared key Azure credential: %w", err)
+		}
+
+		client, err = azblob.NewClientWithSharedKeyCredential(normalizedEndpoint, cred, nil)
+		if err != nil {
+			return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err)
+		}
+	} else {
+		cred, err := azidentity.NewManagedIdentityCredential(nil)
+		if err != nil {
+			return nil, fmt.Errorf("NewStorageBackend: error creating managed identity credential: %w", err)
+		}
+		client, err = azblob.NewClient(normalizedEndpoint, cred, nil)
+		if err != nil {
+			return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err)
+		}
+	}
+
+	storage := azureBlobStorage{
+		client:        client,
+		containerName: opts.ContainerName,
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.RemotePath,
+			Log:             logFunc,
+		},
+	}
+	return &storage, nil
+}
+
+// Name returns the name of the storage backend
+func (b *azureBlobStorage) Name() string {
+	return "Azure"
+}
+
+// Copy copies the given file to the storage backend.
+func (b *azureBlobStorage) Copy(file string) error {
+	fileReader, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err)
+	}
+	_, err = b.client.UploadStream(
+		context.Background(),
+		b.containerName,
+		filepath.Join(b.DestinationPath, filepath.Base(file)),
+		fileReader,
+		nil,
+	)
+	if err != nil {
+		return fmt.Errorf("(*azureBlobStorage).Copy: error uploading file %s: %w", file, err)
+	}
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided
+// deadline for the Azure Blob storage backend.
+func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	lookupPrefix := filepath.Join(b.DestinationPath, pruningPrefix)
+	pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{
+		Prefix: &lookupPrefix,
+	})
+	var matches []string
+	var totalCount uint
+	for pager.More() {
+		resp, err := pager.NextPage(context.Background())
+		if err != nil {
+			return nil, fmt.Errorf("(*azureBlobStorage).Prune: error paging over blobs: %w", err)
+		}
+		for _, v := range resp.Segment.BlobItems {
+			totalCount++
+			if v.Properties.LastModified.Before(deadline) {
+				matches = append(matches, *v.Name)
+			}
+		}
+	}
+
+	stats := storage.PruneStats{
+		Total:  totalCount,
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), int(totalCount), "Azure Blob Storage backup(s)", func() error {
+		wg := sync.WaitGroup{}
+		wg.Add(len(matches))
+		var errs []error
+
+		for _, match := range matches {
+			name := match
+			go func() {
+				_, err := b.client.DeleteBlob(context.Background(), b.containerName, name, nil)
+				if err != nil {
+					errs = append(errs, err)
+				}
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+		if len(errs) != 0 {
+			return errors.Join(errs...)
+		}
+		return nil
+	}); err != nil {
+		return &stats, err
+	}
+
+	return &stats, nil
+}

internal/storage/local/local.go

@@ -0,0 +1,160 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package local
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+)
+
+type localStorage struct {
+	*storage.StorageBackend
+	latestSymlink string
+}
+
+// Config allows configuration of a local storage backend.
+type Config struct {
+	ArchivePath   string
+	LatestSymlink string
+}
+
+// NewStorageBackend creates and initializes a new local storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) storage.Backend {
+	return &localStorage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.ArchivePath,
+			Log:             logFunc,
+		},
+		latestSymlink: opts.LatestSymlink,
+	}
+}
+
+// Name return the name of the storage backend
+func (b *localStorage) Name() string {
+	return "Local"
+}
+
+// Copy copies the given file to the local storage backend.
+func (b *localStorage) Copy(file string) error {
+	_, name := path.Split(file)
+
+	if err := copyFile(file, path.Join(b.DestinationPath, name)); err != nil {
+		return fmt.Errorf("(*localStorage).Copy: Error copying file to local archive: %w", err)
+	}
+	b.Log(storage.LogLevelInfo, b.Name(), "Stored copy of backup `%s` in local archive `%s`.", file, b.DestinationPath)
+
+	if b.latestSymlink != "" {
+		symlink := path.Join(b.DestinationPath, b.latestSymlink)
+		if _, err := os.Lstat(symlink); err == nil {
+			os.Remove(symlink)
+		}
+		if err := os.Symlink(name, symlink); err != nil {
+			return fmt.Errorf("(*localStorage).Copy: error creating latest symlink: %w", err)
+		}
+		b.Log(storage.LogLevelInfo, b.Name(), "Created/Updated symlink `%s` for latest backup.", b.latestSymlink)
+	}
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the local storage backend.
+func (b *localStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	globPattern := path.Join(
+		b.DestinationPath,
+		fmt.Sprintf("%s*", pruningPrefix),
+	)
+	globMatches, err := filepath.Glob(globPattern)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"(*localStorage).Prune: Error looking up matching files using pattern %s: %w",
+			globPattern,
+			err,
+		)
+	}
+
+	var candidates []string
+	for _, candidate := range globMatches {
+		fi, err := os.Lstat(candidate)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"(*localStorage).Prune: Error calling Lstat on file %s: %w",
+				candidate,
+				err,
+			)
+		}
+
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			candidates = append(candidates, candidate)
+		}
+	}
+
+	var matches []string
+	for _, candidate := range candidates {
+		fi, err := os.Stat(candidate)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"(*localStorage).Prune: Error calling stat on file %s: %w",
+				candidate,
+				err,
+			)
+		}
+		if fi.ModTime().Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(len(candidates)),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), len(candidates), "local backup(s)", func() error {
+		var removeErrors []error
+		for _, match := range matches {
+			if err := os.Remove(match); err != nil {
+				removeErrors = append(removeErrors, err)
+			}
+		}
+		if len(removeErrors) != 0 {
+			return fmt.Errorf(
+				"(*localStorage).Prune: %d error(s) deleting local files, starting with: %w",
+				len(removeErrors),
+				errors.Join(removeErrors...),
+			)
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}
+
+// copy creates a copy of the file located at `dst` at `src`.
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}

internal/storage/s3/s3.go

@@ -0,0 +1,169 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package s3
+
+import (
+	"context"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/offen/docker-volume-backup/internal/storage"
+)
+
+type s3Storage struct {
+	*storage.StorageBackend
+	client       *minio.Client
+	bucket       string
+	storageClass string
+}
+
+// Config contains values that define the configuration of a S3 backend.
+type Config struct {
+	Endpoint         string
+	AccessKeyID      string
+	SecretAccessKey  string
+	IamRoleEndpoint  string
+	EndpointProto    string
+	EndpointInsecure bool
+	RemotePath       string
+	BucketName       string
+	StorageClass     string
+	CACert           *x509.Certificate
+}
+
+// NewStorageBackend creates and initializes a new S3/Minio storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	var creds *credentials.Credentials
+	if opts.AccessKeyID != "" && opts.SecretAccessKey != "" {
+		creds = credentials.NewStaticV4(
+			opts.AccessKeyID,
+			opts.SecretAccessKey,
+			"",
+		)
+	} else if opts.IamRoleEndpoint != "" {
+		creds = credentials.NewIAM(opts.IamRoleEndpoint)
+	} else {
+		return nil, errors.New("NewStorageBackend: AWS_S3_BUCKET_NAME is defined, but no credentials were provided")
+	}
+
+	options := minio.Options{
+		Creds:  creds,
+		Secure: opts.EndpointProto == "https",
+	}
+
+	transport, err := minio.DefaultTransport(true)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: failed to create default minio transport: %w", err)
+	}
+
+	if opts.EndpointInsecure {
+		if !options.Secure {
+			return nil, errors.New("NewStorageBackend: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
+		}
+		transport.TLSClientConfig.InsecureSkipVerify = true
+	} else if opts.CACert != nil {
+		if transport.TLSClientConfig.RootCAs == nil {
+			transport.TLSClientConfig.RootCAs = x509.NewCertPool()
+		}
+		transport.TLSClientConfig.RootCAs.AddCert(opts.CACert)
+	}
+	options.Transport = transport
+
+	mc, err := minio.New(opts.Endpoint, &options)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error setting up minio client: %w", err)
+	}
+
+	return &s3Storage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.RemotePath,
+			Log:             logFunc,
+		},
+		client:       mc,
+		bucket:       opts.BucketName,
+		storageClass: opts.StorageClass,
+	}, nil
+}
+
+// Name returns the name of the storage backend
+func (v *s3Storage) Name() string {
+	return "S3"
+}
+
+// Copy copies the given file to the S3/Minio storage backend.
+func (b *s3Storage) Copy(file string) error {
+	_, name := path.Split(file)
+
+	if _, err := b.client.FPutObject(context.Background(), b.bucket, filepath.Join(b.DestinationPath, name), file, minio.PutObjectOptions{
+		ContentType:  "application/tar+gzip",
+		StorageClass: b.storageClass,
+	}); err != nil {
+		if errResp := minio.ToErrorResponse(err); errResp.Message != "" {
+			return fmt.Errorf("(*s3Storage).Copy: error uploading backup to remote storage: [Message]: '%s', [Code]: %s, [StatusCode]: %d", errResp.Message, errResp.Code, errResp.StatusCode)
+		}
+		return fmt.Errorf("(*s3Storage).Copy: error uploading backup to remote storage: %w", err)
+	}
+	b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to bucket `%s`.", file, b.bucket)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend.
+func (b *s3Storage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates := b.client.ListObjects(context.Background(), b.bucket, minio.ListObjectsOptions{
+		Prefix:    filepath.Join(b.DestinationPath, pruningPrefix),
+		Recursive: true,
+	})
+
+	var matches []minio.ObjectInfo
+	var lenCandidates int
+	for candidate := range candidates {
+		lenCandidates++
+		if candidate.Err != nil {
+			return nil, fmt.Errorf(
+				"(*s3Storage).Prune: Error looking up candidates from remote storage! %w",
+				candidate.Err,
+			)
+		}
+		if candidate.LastModified.Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(lenCandidates),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), lenCandidates, "remote backup(s)", func() error {
+		objectsCh := make(chan minio.ObjectInfo)
+		go func() {
+			for _, match := range matches {
+				objectsCh <- match
+			}
+			close(objectsCh)
+		}()
+		errChan := b.client.RemoveObjects(context.Background(), b.bucket, objectsCh, minio.RemoveObjectsOptions{})
+		var removeErrors []error
+		for result := range errChan {
+			if result.Err != nil {
+				removeErrors = append(removeErrors, result.Err)
+			}
+		}
+		if len(removeErrors) != 0 {
+			return errors.Join(removeErrors...)
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}

internal/storage/ssh/ssh.go

@@ -0,0 +1,190 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package ssh
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+type sshStorage struct {
+	*storage.StorageBackend
+	client     *ssh.Client
+	sftpClient *sftp.Client
+	hostName   string
+}
+
+// Config allows to configure a SSH backend.
+type Config struct {
+	HostName           string
+	Port               string
+	User               string
+	Password           string
+	IdentityFile       string
+	IdentityPassphrase string
+	RemotePath         string
+}
+
+// NewStorageBackend creates and initializes a new SSH storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	var authMethods []ssh.AuthMethod
+
+	if opts.Password != "" {
+		authMethods = append(authMethods, ssh.Password(opts.Password))
+	}
+
+	if _, err := os.Stat(opts.IdentityFile); err == nil {
+		key, err := ioutil.ReadFile(opts.IdentityFile)
+		if err != nil {
+			return nil, errors.New("NewStorageBackend: error reading the private key")
+		}
+
+		var signer ssh.Signer
+		if opts.IdentityPassphrase != "" {
+			signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(opts.IdentityPassphrase))
+			if err != nil {
+				return nil, errors.New("NewStorageBackend: error parsing the encrypted private key")
+			}
+			authMethods = append(authMethods, ssh.PublicKeys(signer))
+		} else {
+			signer, err = ssh.ParsePrivateKey(key)
+			if err != nil {
+				return nil, errors.New("NewStorageBackend: error parsing the private key")
+			}
+			authMethods = append(authMethods, ssh.PublicKeys(signer))
+		}
+	}
+
+	sshClientConfig := &ssh.ClientConfig{
+		User:            opts.User,
+		Auth:            authMethods,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	sshClient, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", opts.HostName, opts.Port), sshClientConfig)
+
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: Error creating ssh client: %w", err)
+	}
+	_, _, err = sshClient.SendRequest("keepalive", false, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	sftpClient, err := sftp.NewClient(sshClient)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error creating sftp client: %w", err)
+	}
+
+	return &sshStorage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: opts.RemotePath,
+			Log:             logFunc,
+		},
+		client:     sshClient,
+		sftpClient: sftpClient,
+		hostName:   opts.HostName,
+	}, nil
+}
+
+// Name returns the name of the storage backend
+func (b *sshStorage) Name() string {
+	return "SSH"
+}
+
+// Copy copies the given file to the SSH storage backend.
+func (b *sshStorage) Copy(file string) error {
+	source, err := os.Open(file)
+	_, name := path.Split(file)
+	if err != nil {
+		return fmt.Errorf("(*sshStorage).Copy: Error reading the file to be uploaded: %w", err)
+	}
+	defer source.Close()
+
+	destination, err := b.sftpClient.Create(filepath.Join(b.DestinationPath, name))
+	if err != nil {
+		return fmt.Errorf("(*sshStorage).Copy: Error creating file on SSH storage: %w", err)
+	}
+	defer destination.Close()
+
+	chunk := make([]byte, 1000000)
+	for {
+		num, err := source.Read(chunk)
+		if err == io.EOF {
+			tot, err := destination.Write(chunk[:num])
+			if err != nil {
+				return fmt.Errorf("(*sshStorage).Copy: Error uploading the file to SSH storage: %w", err)
+			}
+
+			if tot != len(chunk[:num]) {
+				return errors.New("(*sshStorage).Copy: failed to write stream")
+			}
+
+			break
+		}
+
+		if err != nil {
+			return fmt.Errorf("(*sshStorage).Copy: Error uploading the file to SSH storage: %w", err)
+		}
+
+		tot, err := destination.Write(chunk[:num])
+		if err != nil {
+			return fmt.Errorf("(*sshStorage).Copy: Error uploading the file to SSH storage: %w", err)
+		}
+
+		if tot != len(chunk[:num]) {
+			return fmt.Errorf("(*sshStorage).Copy: failed to write stream")
+		}
+	}
+
+	b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to SSH storage '%s' at path '%s'.", file, b.hostName, b.DestinationPath)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the SSH storage backend.
+func (b *sshStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates, err := b.sftpClient.ReadDir(b.DestinationPath)
+	if err != nil {
+		return nil, fmt.Errorf("(*sshStorage).Prune: Error reading directory from SSH storage: %w", err)
+	}
+
+	var matches []string
+	for _, candidate := range candidates {
+		if !strings.HasPrefix(candidate.Name(), pruningPrefix) {
+			continue
+		}
+		if candidate.ModTime().Before(deadline) {
+			matches = append(matches, candidate.Name())
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(len(candidates)),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), len(candidates), "SSH backup(s)", func() error {
+		for _, match := range matches {
+			if err := b.sftpClient.Remove(filepath.Join(b.DestinationPath, match)); err != nil {
+				return fmt.Errorf("(*sshStorage).Prune: Error removing file from SSH storage: %w", err)
+			}
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}

internal/storage/storage.go

@@ -0,0 +1,61 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package storage
+
+import (
+	"time"
+)
+
+// Backend is an interface for defining functions which all storage providers support.
+type Backend interface {
+	Copy(file string) error
+	Prune(deadline time.Time, pruningPrefix string) (*PruneStats, error)
+	Name() string
+}
+
+// StorageBackend is a generic type of storage. Everything here are common properties of all storage types.
+type StorageBackend struct {
+	DestinationPath string
+	RetentionDays   int
+	Log             Log
+}
+
+type LogLevel int
+
+const (
+	LogLevelInfo LogLevel = iota
+	LogLevelWarning
+	LogLevelError
+)
+
+type Log func(logType LogLevel, context string, msg string, params ...any)
+
+// PruneStats is a wrapper struct for returning stats after pruning
+type PruneStats struct {
+	Total  uint
+	Pruned uint
+}
+
+// DoPrune holds general control flow that applies to any kind of storage.
+// Callers can pass in a thunk that performs the actual deletion of files.
+func (b *StorageBackend) DoPrune(context string, lenMatches, lenCandidates int, description string, doRemoveFiles func() error) error {
+	if lenMatches != 0 && lenMatches != lenCandidates {
+		if err := doRemoveFiles(); err != nil {
+			return err
+		}
+		b.Log(LogLevelInfo, context,
+			"Pruned %d out of %d %s as their age exceeded the configured retention period of %d days.",
+			lenMatches,
+			lenCandidates,
+			description,
+			b.RetentionDays,
+		)
+	} else if lenMatches != 0 && lenMatches == lenCandidates {
+		b.Log(LogLevelWarning, context, "The current configuration would delete all %d existing %s.", lenMatches, description)
+		b.Log(LogLevelWarning, context, "Refusing to do so, please check your configuration.")
+	} else {
+		b.Log(LogLevelInfo, context, "None of %d existing %s were pruned.", lenCandidates, description)
+	}
+	return nil
+}

internal/storage/webdav/webdav.go

@@ -0,0 +1,123 @@
+// Copyright 2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package webdav
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/studio-b12/gowebdav"
+)
+
+type webDavStorage struct {
+	*storage.StorageBackend
+	client *gowebdav.Client
+	url    string
+}
+
+// Config allows to configure a WebDAV storage backend.
+type Config struct {
+	URL         string
+	RemotePath  string
+	Username    string
+	Password    string
+	URLInsecure bool
+}
+
+// NewStorageBackend creates and initializes a new WebDav storage backend.
+func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
+	if opts.Username == "" || opts.Password == "" {
+		return nil, errors.New("NewStorageBackend: WEBDAV_URL is defined, but no credentials were provided")
+	} else {
+		webdavClient := gowebdav.NewClient(opts.URL, opts.Username, opts.Password)
+
+		if opts.URLInsecure {
+			defaultTransport, ok := http.DefaultTransport.(*http.Transport)
+			if !ok {
+				return nil, errors.New("NewStorageBackend: unexpected error when asserting type for http.DefaultTransport")
+			}
+			webdavTransport := defaultTransport.Clone()
+			webdavTransport.TLSClientConfig.InsecureSkipVerify = opts.URLInsecure
+			webdavClient.SetTransport(webdavTransport)
+		}
+
+		return &webDavStorage{
+			StorageBackend: &storage.StorageBackend{
+				DestinationPath: opts.RemotePath,
+				Log:             logFunc,
+			},
+			client: webdavClient,
+		}, nil
+	}
+}
+
+// Name returns the name of the storage backend
+func (b *webDavStorage) Name() string {
+	return "WebDAV"
+}
+
+// Copy copies the given file to the WebDav storage backend.
+func (b *webDavStorage) Copy(file string) error {
+	_, name := path.Split(file)
+	if err := b.client.MkdirAll(b.DestinationPath, 0644); err != nil {
+		return fmt.Errorf("(*webDavStorage).Copy: Error creating directory '%s' on WebDAV server: %w", b.DestinationPath, err)
+	}
+
+	r, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("(*webDavStorage).Copy: Error opening the file to be uploaded: %w", err)
+	}
+
+	if err := b.client.WriteStream(filepath.Join(b.DestinationPath, name), r, 0644); err != nil {
+		return fmt.Errorf("(*webDavStorage).Copy: Error uploading the file to WebDAV server: %w", err)
+	}
+	b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup '%s' to WebDAV URL '%s' at path '%s'.", file, b.url, b.DestinationPath)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the WebDav storage backend.
+func (b *webDavStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates, err := b.client.ReadDir(b.DestinationPath)
+	if err != nil {
+		return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err)
+	}
+	var matches []fs.FileInfo
+	var lenCandidates int
+	for _, candidate := range candidates {
+		if !strings.HasPrefix(candidate.Name(), pruningPrefix) {
+			continue
+		}
+		lenCandidates++
+		if candidate.ModTime().Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(lenCandidates),
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), lenCandidates, "WebDAV backup(s)", func() error {
+		for _, match := range matches {
+			if err := b.client.Remove(filepath.Join(b.DestinationPath, match.Name())); err != nil {
+				return fmt.Errorf("(*webDavStorage).Prune: Error removing file from WebDAV storage: %w", err)
+			}
+		}
+		return nil
+	}); err != nil {
+		return stats, err
+	}
+
+	return stats, nil
+}

src/backup.sh

@@ -1,150 +0,0 @@
-#!/bin/sh
-
-# Copyright 2021 - Offen Authors <hioffen@posteo.de>
-# SPDX-License-Identifier: MPL-2.0
-
-# Portions of this file are taken from github.com/futurice/docker-volume-backup
-# See NOTICE for information about authors and licensing.
-
-source env.sh
-
-function info {
-  echo -e "\n[INFO] $1\n"
-}
-
-info "Preparing backup"
-DOCKER_SOCK="/var/run/docker.sock"
-
-if [ -S "$DOCKER_SOCK" ]; then
-  TEMPFILE="$(mktemp)"
-  docker ps -q \
-    --filter "label=docker-volume-backup.stop-during-backup=$BACKUP_STOP_CONTAINER_LABEL" \
-    > "$TEMPFILE"
-  CONTAINERS_TO_STOP="$(cat $TEMPFILE | tr '\n' ' ')"
-  CONTAINERS_TO_STOP_TOTAL="$(cat $TEMPFILE | wc -l)"
-  CONTAINERS_TOTAL="$(docker ps -q | wc -l)"
-  rm "$TEMPFILE"
-  echo "$CONTAINERS_TOTAL containers running on host in total."
-  echo "$CONTAINERS_TO_STOP_TOTAL containers marked to be stopped during backup."
-else
-  CONTAINERS_TO_STOP_TOTAL="0"
-  CONTAINERS_TOTAL="0"
-  echo "Cannot access \"$DOCKER_SOCK\", won't look for containers to stop."
-fi
-
-if [ "$CONTAINERS_TO_STOP_TOTAL" != "0" ]; then
-  info "Stopping containers"
-  docker stop $CONTAINERS_TO_STOP
-fi
-
-info "Creating backup"
-BACKUP_FILENAME="$(date +"$BACKUP_FILENAME")"
-tar -czvf "$BACKUP_FILENAME" $BACKUP_SOURCES # allow the var to expand, in case we have multiple sources
-
-if [ ! -z "$GPG_PASSPHRASE" ]; then
-  info "Encrypting backup"
-  gpg --symmetric --cipher-algo aes256 --batch --passphrase "$GPG_PASSPHRASE" \
-    -o "${BACKUP_FILENAME}.gpg" $BACKUP_FILENAME
-  rm $BACKUP_FILENAME
-  BACKUP_FILENAME="${BACKUP_FILENAME}.gpg"
-fi
-
-if [ "$CONTAINERS_TO_STOP_TOTAL" != "0" ]; then
-  info "Starting containers/services back up"
-  # The container might be part of a stack when running in swarm mode, so
-  # its parent service needs to be restarted instead once backup is finished.
-  SERVICES_REQUIRING_UPDATE=""
-  for CONTAINER_ID in $CONTAINERS_TO_STOP; do
-    SWARM_SERVICE_NAME=$(
-      docker inspect \
-        --format "{{ index .Config.Labels \"com.docker.swarm.service.name\" }}" \
-        $CONTAINER_ID
-    )
-    if [ -z "$SWARM_SERVICE_NAME" ]; then
-      echo "Restarting $(docker start $CONTAINER_ID)"
-    else
-      echo "Removing $(docker rm $CONTAINER_ID)"
-      # Multiple containers might belong to the same service, so they will
-      # be restarted only after all names are known.
-      SERVICES_REQUIRING_UPDATE="${SERVICES_REQUIRING_UPDATE} ${SWARM_SERVICE_NAME}"
-    fi
-  done
-
-  if [ -n "$SERVICES_REQUIRING_UPDATE" ]; then
-    for SERVICE_NAME in $(echo -n "$SERVICES_REQUIRING_UPDATE" | tr ' ' '\n' | sort -u); do
-      docker service update --force $SERVICE_NAME
-    done
-  fi
-fi
-
-copy_backup () {
-  mc cp $MC_GLOBAL_OPTIONS "$BACKUP_FILENAME" "$1"
-}
-
-if [ ! -z "$AWS_S3_BUCKET_NAME" ]; then
-  info "Uploading backup to remote storage"
-  echo "Will upload to bucket \"$AWS_S3_BUCKET_NAME\"."
-  copy_backup "backup-target/$AWS_S3_BUCKET_NAME"
-  echo "Upload finished."
-fi
-
-if [ -d "$BACKUP_ARCHIVE" ]; then
-  info "Copying backup to local archive"
-  echo "Will copy to \"$BACKUP_ARCHIVE\"."
-  copy_backup "$BACKUP_ARCHIVE"
-  echo "Finished copying."
-fi
-
-if [ -f "$BACKUP_FILENAME" ]; then
-  info "Cleaning up"
-  rm -vf "$BACKUP_FILENAME"
-fi
-
-info "Backup finished"
-echo "Will wait for next scheduled backup."
-
-prune () {
-  target=$1
-  if [ ! -z "$BACKUP_PRUNING_PREFIX" ]; then
-    target="$target/${BACKUP_PRUNING_PREFIX}"
-  fi
-
-  rule_applies_to=$(
-    mc rm $MC_GLOBAL_OPTIONS --fake --recursive --force \
-      --older-than "${BACKUP_RETENTION_DAYS}d" \
-      "$target" \
-      | wc -l
-  )
-  if [ "$rule_applies_to" == "0" ]; then
-    echo "No backups found older than the configured retention period of $BACKUP_RETENTION_DAYS days."
-    echo "Doing nothing."
-    exit 0
-  fi
-
-  total=$(mc ls $MC_GLOBAL_OPTIONS "$target" | wc -l)
-
-  if [ "$rule_applies_to" == "$total" ]; then
-    echo "Using a retention of ${BACKUP_RETENTION_DAYS} days would prune all currently existing backups, will not continue."
-    echo "If this is what you want, please remove files manually instead of using this script."
-    exit 1
-  fi
-
-  mc rm $MC_GLOBAL_OPTIONS \
-    --recursive --force \
-    --older-than "${BACKUP_RETENTION_DAYS}d" "$target"
-  echo "Successfully pruned ${rule_applies_to} backups older than ${BACKUP_RETENTION_DAYS} days."
-}
-
-if [ ! -z "$BACKUP_RETENTION_DAYS" ]; then
-  info "Pruning old backups"
-  echo "Sleeping ${BACKUP_PRUNING_LEEWAY} before checking eligibility."
-  sleep "$BACKUP_PRUNING_LEEWAY"
-  if [ ! -z "$AWS_S3_BUCKET_NAME" ]; then
-    info "Pruning old backups from remote storage"
-    prune "backup-target/$bucket"
-  fi
-  if [ -d "$BACKUP_ARCHIVE" ]; then
-    info "Pruning old backups from local archive"
-    prune "$BACKUP_ARCHIVE"
-  fi
-fi

src/entrypoint.sh

@@ -1,46 +0,0 @@
-#!/bin/sh
-
-# Copyright 2021 - Offen Authors <hioffen@posteo.de>
-# SPDX-License-Identifier: MPL-2.0
-
-# Portions of this file are taken from github.com/futurice/docker-volume-backup
-# See NOTICE for information about authors and licensing.
-
-set -e
-
-# Write cronjob env to file, fill in sensible defaults, and read them back in
-cat <<EOF > env.sh
-BACKUP_SOURCES="${BACKUP_SOURCES:-/backup}"
-BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
-BACKUP_FILENAME="${BACKUP_FILENAME:-backup-%Y-%m-%dT%H-%M-%S.tar.gz}"
-BACKUP_ARCHIVE="${BACKUP_ARCHIVE:-/archive}"
-
-BACKUP_RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-}"
-BACKUP_PRUNING_LEEWAY="${BACKUP_PRUNING_LEEWAY:-10m}"
-BACKUP_PRUNING_PREFIX="${BACKUP_PRUNING_PREFIX:-}"
-BACKUP_STOP_CONTAINER_LABEL="${BACKUP_STOP_CONTAINER_LABEL:-true}"
-
-AWS_S3_BUCKET_NAME="${AWS_S3_BUCKET_NAME:-}"
-AWS_ENDPOINT="${AWS_ENDPOINT:-s3.amazonaws.com}"
-AWS_ENDPOINT_PROTO="${AWS_ENDPOINT_PROTO:-https}"
-
-GPG_PASSPHRASE="${GPG_PASSPHRASE:-}"
-
-MC_GLOBAL_OPTIONS="${MC_GLOBAL_OPTIONS:-}"
-EOF
-chmod a+x env.sh
-source env.sh
-
-if [ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ]; then
-  mc $MC_GLOBAL_OPTIONS alias set backup-target \
-    "$AWS_ENDPOINT_PROTO://$AWS_ENDPOINT" \
-    "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
-fi
-
-# Add our cron entry, and direct stdout & stderr to Docker commands stdout
-echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
-echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
-
-# Let cron take the wheel
-echo "Starting cron in foreground."
-crond -f -l 8

test/azure/docker-compose.yml

@@ -0,0 +1,58 @@
+version: '3'
+
+services:
+  storage:
+    image: mcr.microsoft.com/azure-storage/azurite
+    volumes:
+      - azurite_backup_data:/data
+    command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data
+    healthcheck:
+        test: nc 127.0.0.1 10000 -z
+        interval: 1s
+        retries: 30
+
+  az_cli:
+    image: mcr.microsoft.com/azure-cli
+    volumes:
+      - ./local:/dump
+    command:
+      - /bin/sh
+      - -c
+      - |
+          az storage container create --name test-container
+    depends_on:
+      storage:
+        condition: service_healthy
+    environment:
+      AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1;
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    restart: always
+    environment:
+      AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1
+      AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      AZURE_STORAGE_CONTAINER_NAME: test-container
+      AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/
+      AZURE_STORAGE_PATH: 'path/to/backup'
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  azurite_backup_data:
+    name: azurite_backup_data
+  app_data:

test/azure/run.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker compose up -d
+sleep 5
+
+# A symlink for a known file in the volume is created so the test can check
+# whether symlinks are preserved on backup.
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+docker-compose run --rm az_cli \
+  az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz
+tar -xvf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
+
+pass "Found relevant files in untared remote backups."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker-compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+docker compose run --rm az_cli \
+  az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz
+test -f ./local/test.tar.gz
+
+pass "Remote backups have not been deleted."
+
+docker compose down --volumes

test/certs/docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3'
+
+services:
+  minio:
+    hostname: minio.local
+    image: minio/minio:RELEASE.2020-08-04T23-10-51Z
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server --certs-dir "/certs" --address ":443" /data'
+    volumes:
+      - minio_backup_data:/data
+      - ./minio.crt:/certs/public.crt
+      - ./minio.key:/certs/private.key
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    depends_on:
+      - minio
+    restart: always
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ
+      AWS_ENDPOINT: minio.local:443
+      AWS_ENDPOINT_CA_CERT: /root/minio-rootCA.crt
+      AWS_S3_BUCKET_NAME: backup
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./rootCA.crt:/root/minio-rootCA.crt
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  minio_backup_data:
+    name: minio_backup_data
+  app_data:

test/certs/run.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+openssl genrsa -des3 -passout pass:test -out rootCA.key 4096
+openssl req -passin pass:test \
+  -subj "/C=DE/ST=BE/O=IntegrationTest, Inc." \
+  -x509 -new -key rootCA.key -sha256 -days 1 -out rootCA.crt
+
+openssl genrsa -out minio.key 4096
+openssl req -new -sha256 -key minio.key \
+  -subj "/C=DE/ST=BE/O=IntegrationTest, Inc./CN=minio" \
+  -out minio.csr
+
+openssl x509 -req -passin pass:test \
+  -in minio.csr \
+  -CA rootCA.crt -CAkey rootCA.key -CAcreateserial \
+  -extfile san.cnf \
+  -out minio.crt -days 1 -sha256
+
+openssl x509 -in minio.crt -noout -text
+
+docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'tar -xvf /minio_data/backup/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in untared remote backups."
+
+docker compose down --volumes

test/certs/san.cnf

@@ -0,0 +1 @@
+subjectAltName = DNS:minio.local

test/cli/run.sh

@@ -3,10 +3,15 @@
 set -e
 
 cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
 
 docker network create test_network
 docker volume create backup_data
 docker volume create app_data
+# This volume is created to test whether empty directories are handled
+# correctly. It is not supposed to hold any data.
+docker volume create empty_data
 
 docker run -d \
   --name minio \
@@ -23,16 +28,15 @@ docker exec minio mkdir -p /data/backup
 docker run -d \
   --name offen \
   --network test_network \
-  --label "docker-volume-backup.stop-during-backup=true" \
   -v app_data:/var/opt/offen/ \
   offen/offen:latest
 
 sleep 10
 
-docker run -d \
-  --name backup \
+docker run --rm \
   --network test_network \
   -v app_data:/backup/app_data \
+  -v empty_data:/backup/empty_data \
   -v /var/run/docker.sock:/var/run/docker.sock \
   --env AWS_ACCESS_KEY_ID=test \
   --env AWS_SECRET_ACCESS_KEY=GMusLtUmILge2by+z890kQ \
@@ -40,25 +44,20 @@ docker run -d \
   --env AWS_ENDPOINT_PROTO=http \
   --env AWS_S3_BUCKET_NAME=backup \
   --env BACKUP_FILENAME=test.tar.gz \
-  --env BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?" \
-  offen/docker-volume-backup:$TEST_VERSION
+  --env "BACKUP_FROM_SNAPSHOT=true" \
+  --entrypoint backup \
+  offen/docker-volume-backup:${TEST_VERSION:-canary}
 
-docker exec backup backup
-
-docker run --rm -it \
+docker run --rm \
   -v backup_data:/data alpine \
-  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
+  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db && test -d /backup/empty_data'
 
-echo "[TEST:PASS] Found relevant files in untared backup."
+pass "Found relevant files in untared remote backup."
 
-if [ "$(docker ps -q | wc -l)" != "3" ]; then
-  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
-  docker ps
-  exit 1
-fi
+# This test does not stop containers during backup. This is happening on
+# purpose in order to cover this setup as well.
+expect_running_containers "2"
 
-echo "[TEST:PASS] All containers running post backup."
-
-docker rm $(docker stop minio offen backup)
+docker rm $(docker stop minio offen)
 docker volume rm backup_data app_data
 docker network rm test_network

test/commands/docker-compose.yml

@@ -0,0 +1,51 @@
+version: '3.8'
+
+services:
+  database:
+    image: mariadb:10.7
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: backup
+    labels:
+      # this is testing the deprecated label on purpose
+      - docker-volume-backup.exec-pre=/bin/sh -c 'mysqldump -ptest --all-databases > /tmp/volume/dump.sql'
+      - docker-volume-backup.copy-post=/bin/sh -c 'echo "post" > /tmp/volume/post.txt'
+      - docker-volume-backup.exec-label=test
+    volumes:
+      - app_data:/tmp/volume
+
+  other_database:
+    image: mariadb:10.7
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: backup
+    labels:
+      - docker-volume-backup.archive-pre=touch /tmp/volume/not-relevant.txt
+      - docker-volume-backup.exec-label=not-relevant
+    volumes:
+      - app_data:/tmp/volume
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      EXEC_LABEL: test
+      EXEC_FORWARD_OUTPUT: "true"
+    volumes:
+      - archive:/archive
+      - app_data:/backup/data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  app_data:
+  archive:

test/commands/run.sh

@@ -0,0 +1,64 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker compose up -d
+sleep 30 # mariadb likes to take a bit before responding
+
+docker compose exec backup backup
+sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' commands_archive) ./local
+
+tar -xvf ./local/test.tar.gz
+if [ ! -f ./backup/data/dump.sql ]; then
+  fail "Could not find file written by pre command."
+fi
+pass "Found expected file."
+
+if [ -f ./backup/data/not-relevant.txt ]; then
+  fail "Command ran for container with other label."
+fi
+pass "Command did not run for container with other label."
+
+if [ -f ./backup/data/post.txt ]; then
+  fail "File created in post command was present in backup."
+fi
+pass "Did not find unexpected file."
+
+docker compose down --volumes
+sudo rm -rf ./local
+
+
+info "Running commands test in swarm mode next."
+
+docker swarm init
+
+docker stack deploy --compose-file=docker-compose.yml test_stack
+
+while [ -z $(docker ps -q -f name=backup) ]; do
+  info "Backup container not ready yet. Retrying."
+  sleep 1
+done
+
+sleep 20
+
+docker exec $(docker ps -q -f name=backup) backup
+
+sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' test_stack_archive) ./local
+
+tar -xvf ./local/test.tar.gz
+if [ ! -f ./backup/data/dump.sql ]; then
+  fail "Could not find file written by pre command."
+fi
+pass "Found expected file."
+
+if [ -f ./backup/data/post.txt ]; then
+  fail "File created in post command was present in backup."
+fi
+pass "Did not find unexpected file."
+
+docker stack rm test_stack
+docker swarm leave --force

test/compose/run.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(dirname $0)
-
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-docker run --rm -it \
-  -v compose_backup_data:/data alpine \
-  ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
-
-echo "[TEST:PASS] Found relevant files in untared remote backup."
-
-tar -xf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
-
-echo "[TEST:PASS] Found relevant files in untared local backup."
-
-if [ "$(docker-compose ps -q | wc -l)" != "3" ]; then
-  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
-  docker-compose ps
-  exit 1
-fi
-
-echo "[TEST:PASS] All containers running post backup."
-
-docker-compose down --volumes

test/confd/.gitignore

@@ -0,0 +1 @@
+local

test/confd/01backup.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="conf.tar.gz"
+BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/02backup.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="other.tar.gz"
+BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/03never.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="never.tar.gz"
+BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?"

test/confd/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    volumes:
+      - ./local:/archive
+      - app_data:/backup/app_data:ro
+      - ./01backup.env:/etc/dockervolumebackup/conf.d/01backup.env
+      - ./02backup.env:/etc/dockervolumebackup/conf.d/02backup.env
+      - ./03never.env:/etc/dockervolumebackup/conf.d/03never.env
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/confd/run.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker compose up -d
+
+# sleep until a backup is guaranteed to have happened on the 1 minute schedule
+sleep 100
+
+docker compose down --volumes
+
+if [ ! -f ./local/conf.tar.gz ]; then
+  fail "Config from file was not used."
+fi
+pass "Config from file was used."
+
+if [ ! -f ./local/other.tar.gz ]; then
+  fail "Run on same schedule did not succeed."
+fi
+pass "Run on same schedule succeeded."
+
+if [ -f ./local/never.tar.gz ]; then
+  fail "Unexpected file was found."
+fi
+pass "Unexpected cron did not run."

test/extend/Dockerfile

@@ -0,0 +1,4 @@
+ARG version=canary
+FROM offen/docker-volume-backup:$version
+
+RUN apk add rsync

test/extend/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    labels:
+      - docker-volume-backup.copy-post=/bin/sh -c 'mkdir -p /tmp/unpack && tar -xvf $$COMMAND_RUNTIME_ARCHIVE_FILEPATH -C /tmp/unpack && rsync -r /tmp/unpack/backup/app_data /local'
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      EXEC_FORWARD_OUTPUT: "true"
+    volumes:
+      - ./local:/local
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/extend/run.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+export BASE_VERSION="${TEST_VERSION:-canary}"
+export TEST_VERSION="${TEST_VERSION:-canary}-with-rsync"
+
+docker build . -t offen/docker-volume-backup:$TEST_VERSION --build-arg version=$BASE_VERSION
+
+docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "2"
+
+if [ ! -f "./local/app_data/offen.db" ]; then
+  fail "Could not find expected file in untared archive."
+fi
+
+docker compose down --volumes

test/gpg/.gitignore

@@ -0,0 +1 @@
+local

test/gpg/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    environment:
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      GPG_PASSPHRASE: 1234#$$ecret
+    volumes:
+      - ./local:/archive
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/gpg/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+expect_running_containers "2"
+
+tmp_dir=$(mktemp -d)
+
+echo "1234#\$ecret" | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 ./local/test.tar.gz.gpg > ./local/decrypted.tar.gz
+tar -xf ./local/decrypted.tar.gz -C $tmp_dir
+if [ ! -f $tmp_dir/backup/app_data/offen.db ]; then
+  fail "Could not find expected file in untared archive."
+fi
+rm ./local/decrypted.tar.gz
+
+pass "Found relevant files in decrypted and untared local backup."
+
+if [ ! -L ./local/test-latest.tar.gz.gpg ]; then
+  fail "Could not find local symlink to latest encrypted backup."
+fi
+
+docker compose down --volumes

test/ignore/.gitignore

@@ -0,0 +1 @@
+local

test/ignore/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_EXCLUDE_REGEXP: '\.(me|you)$$'
+    volumes:
+      - ./local:/archive
+      - ./sources:/backup/data:ro

test/ignore/run.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker compose up -d
+sleep 5
+docker compose exec backup backup
+
+docker compose down --volumes
+
+out=$(mktemp -d)
+sudo tar --same-owner -xvf ./local/test.tar.gz -C "$out"
+
+if [ ! -f "$out/backup/data/me.txt" ]; then
+  fail "Expected file was not found."
+fi
+pass "Expected file was found."
+
+if [ -f "$out/backup/data/skip.me" ]; then
+  fail "Ignored file was found."
+fi
+pass "Ignored file was not found."

test/local/.gitignore

@@ -0,0 +1 @@
+local

test/local/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./local:/archive
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/local/run.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker compose up -d
+sleep 5
+
+# A symlink for a known file in the volume is created so the test can check
+# whether symlinks are preserved on backup.
+docker compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "2"
+
+tmp_dir=$(mktemp -d)
+tar -xvf ./local/test-hostnametoken.tar.gz -C $tmp_dir
+if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then
+  fail "Could not find expected file in untared archive."
+fi
+rm -f ./local/test-hostnametoken.tar.gz
+
+if [ ! -L "$tmp_dir/backup/app_data/db.link" ]; then
+  fail "Could not find expected symlink in untared archive."
+fi
+
+pass "Found relevant files in decrypted and untared local backup."
+
+if [ ! -L ./local/test-hostnametoken.latest.tar.gz.gpg ]; then
+  fail "Could not find symlink to latest version."
+fi
+
+pass "Found symlink to latest version in local backup."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+if [ "$(find ./local -type f | wc -l)" != "1" ]; then
+  fail "Backups should not have been deleted, instead seen: "$(find ./local -type f)""
+fi
+pass "Local backups have not been deleted."
+
+docker compose down --volumes

test/notifications/.gitignore

@@ -0,0 +1 @@
+local

test/notifications/docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_PRUNING_PREFIX: test
+      NOTIFICATION_LEVEL: info
+      NOTIFICATION_URLS: ${NOTIFICATION_URLS}
+      EXTRA_VALUE: extra-value
+    volumes:
+      - ./local:/archive
+      - app_data:/backup/app_data:ro
+      - ./notifications.tmpl:/etc/dockervolumebackup/notifications.d/notifications.tmpl
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+  gotify:
+    image: gotify/server
+    ports:
+      - 8080:80
+    environment:
+      - GOTIFY_DEFAULTUSER_PASS=custom
+    volumes:
+      - gotify_data:/app/data
+
+volumes:
+  app_data:
+  gotify_data:

test/notifications/notifications.tmpl

@@ -0,0 +1,7 @@
+{{ define "title_success" -}}
+Successful test run with {{ env "EXTRA_VALUE" }}, yay!
+{{- end }}
+
+{{ define "body_success" -}}
+Backing up {{ .Stats.BackupFile.FullPath }} succeeded.
+{{- end }}

test/notifications/run.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker compose up -d
+sleep 5
+
+GOTIFY_TOKEN=$(curl -sSLX POST -H 'Content-Type: application/json' -d '{"name":"test"}' http://admin:custom@localhost:8080/application | jq -r '.token')
+info "Set up Gotify application using token $GOTIFY_TOKEN"
+
+docker compose exec backup backup
+
+NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
+if [ "$NUM_MESSAGES" != 0 ]; then
+  fail "Expected no notifications to be sent when not configured"
+fi
+pass "No notifications were sent when not configured."
+
+docker compose down
+
+NOTIFICATION_URLS="gotify://gotify/${GOTIFY_TOKEN}?disableTLS=true" docker compose up -d
+
+docker compose exec backup backup
+
+NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
+if [ "$NUM_MESSAGES" != 1 ]; then
+  fail "Expected one notifications to be sent when configured"
+fi
+pass "Correct number of notifications were sent when configured."
+
+MESSAGE_TITLE=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].title')
+MESSAGE_BODY=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].message')
+
+if [ "$MESSAGE_TITLE" != "Successful test run with extra-value, yay!" ]; then
+  fail "Unexpected notification title $MESSAGE_TITLE"
+fi
+pass "Custom notification title was used."
+
+if [ "$MESSAGE_BODY" != "Backing up /tmp/test.tar.gz succeeded." ]; then
+  fail "Unexpected notification body $MESSAGE_BODY"
+fi
+pass "Custom notification body was used."
+
+docker compose down --volumes

test/ownership/.gitignore

@@ -0,0 +1 @@
+local

test/ownership/docker-compose.yml

@@ -0,0 +1,27 @@
+version: '3'
+
+services:
+  db:
+    image: postgres:14-alpine
+    restart: unless-stopped
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=1FHJMSwt0yhIN1zS7I4DilGUhThBKq0x
+      - POSTGRES_USER=test
+      - POSTGRES_DB=test
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION}
+    restart: always
+    environment:
+      BACKUP_FILENAME: backup.tar.gz
+    volumes:
+      - postgres_data:/backup/postgres:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./local:/archive
+
+volumes:
+  postgres_data:

test/ownership/run.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# This test refers to https://github.com/offen/docker-volume-backup/issues/71
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+tmp_dir=$(mktemp -d)
+sudo tar --same-owner -xvf ./local/backup.tar.gz -C $tmp_dir
+
+sudo find $tmp_dir/backup/postgres > /dev/null
+pass "Backup contains files at expected location"
+
+for file in $(sudo find $tmp_dir/backup/postgres); do
+  if [ "$(sudo stat -c '%u:%g' $file)" != "70:70" ]; then
+    fail "Unexpected file ownership for $file: $(sudo stat -c '%u:%g' $file)"
+  fi
+done
+pass "All files and directories in backup preserved their ownership."
+
+docker compose down --volumes

test/s3/docker-compose.yml

@@ -10,10 +10,11 @@ services:
       MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
     entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
     volumes:
-      - backup_data:/data
+      - minio_backup_data:/data
 
-  backup: &default_backup_service
-    image: offen/docker-volume-backup:${TEST_VERSION}
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
     depends_on:
       - minio
     restart: always
@@ -23,10 +24,13 @@ services:
       AWS_ENDPOINT: minio:9000
       AWS_ENDPOINT_PROTO: http
       AWS_S3_BUCKET_NAME: backup
-      BACKUP_FILENAME: test.tar.gz
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
     volumes:
-      - ./local:/archive
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 
@@ -37,7 +41,7 @@ services:
     volumes:
       - app_data:/var/opt/offen
 
-
 volumes:
-  backup_data:
+  minio_backup_data:
+    name: minio_backup_data
   app_data:

test/s3/run.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker compose up -d
+sleep 5
+
+# A symlink for a known file in the volume is created so the test can check
+# whether symlinks are preserved on backup.
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'tar -xvf /minio_data/backup/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in untared remote backups."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+docker run --rm \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c '[ $(find /minio_data/backup/ -type f | wc -l) = "1" ]'
+
+pass "Remote backups have not been deleted."
+
+docker compose down --volumes

test/secrets/docker-compose.yml

@@ -0,0 +1,78 @@
+# Copyright 2020-2021 - Offen Authors <hioffen@posteo.de>
+# SPDX-License-Identifier: Unlicense
+
+version: '3.8'
+
+services:
+  minio:
+    image: minio/minio:RELEASE.2020-08-04T23-10-51Z
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
+    volumes:
+      - backup_data:/data
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    depends_on:
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_root_user
+      AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_root_password
+      AWS_ENDPOINT: minio:9000
+      AWS_ENDPOINT_PROTO: http
+      AWS_S3_BUCKET_NAME: backup
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: 7
+      BACKUP_PRUNING_LEEWAY: 5s
+    volumes:
+      - pg_data:/backup/pg_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    secrets:
+      - minio_root_user
+      - minio_root_password
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    healthcheck:
+      disable: true
+    deploy:
+      replicas: 2
+      restart_policy:
+        condition: on-failure
+
+  pg:
+    image: postgres:14-alpine
+    environment:
+      POSTGRES_PASSWORD: example
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    deploy:
+      restart_policy:
+        condition: on-failure
+
+volumes:
+  backup_data:
+    name: backup_data
+  pg_data:
+    name: pg_data
+
+secrets:
+  minio_root_user:
+    external: true
+  minio_root_password:
+    external: true

test/secrets/run.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker swarm init
+
+printf "test" | docker secret create minio_root_user -
+printf "GMusLtUmILge2by+z890kQ" | docker secret create minio_root_password -
+
+docker stack deploy --compose-file=docker-compose.yml test_stack
+
+while [ -z $(docker ps -q -f name=backup) ]; do
+  info "Backup container not ready yet. Retrying."
+  sleep 1
+done
+
+sleep 20
+
+docker exec $(docker ps -q -f name=backup) backup
+
+docker run --rm \
+  -v backup_data:/data alpine \
+  ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
+
+pass "Found relevant files in untared backup."
+
+sleep 5
+expect_running_containers "5"
+
+docker stack rm test_stack
+
+docker secret rm minio_root_password
+docker secret rm minio_root_user
+
+docker swarm leave --force
+
+sleep 10
+
+docker volume rm backup_data
+docker volume rm pg_data

test/ssh/docker-compose.yml

@@ -0,0 +1,47 @@
+version: '3'
+
+services:
+  ssh:
+    image: linuxserver/openssh-server:version-8.6_p1-r3
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - USER_NAME=test
+    volumes:
+      - ./id_rsa.pub:/config/.ssh/authorized_keys
+      - ssh_backup_data:/tmp
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    depends_on:
+      - ssh
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      SSH_HOST_NAME: ssh
+      SSH_PORT: 2222
+      SSH_USER: test
+      SSH_REMOTE_PATH: /tmp
+      SSH_IDENTITY_PASSPHRASE: test1234
+    volumes:
+      - ./id_rsa:/root/.ssh/id_rsa
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  ssh_backup_data:
+    name: ssh_backup_data
+  app_data:

test/ssh/run.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+ssh-keygen -t rsa -m pem -b 4096 -N "test1234" -f id_rsa -C "docker-volume-backup@local"
+
+docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers 3
+
+docker run --rm \
+  -v ssh_backup_data:/ssh_data \
+  alpine \
+  ash -c 'tar -xvf /ssh_data/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in decrypted and untared remote backups."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+docker run --rm \
+  -v ssh_backup_data:/ssh_data \
+  alpine \
+  ash -c '[ $(find /ssh_data/ -type f | wc -l) = "1" ]'
+
+pass "Remote backups have not been deleted."
+
+docker compose down --volumes
+rm -f id_rsa id_rsa.pub

test/swarm/docker-compose.yml

@@ -18,8 +18,8 @@ services:
     volumes:
       - backup_data:/data
 
-  backup: &default_backup_service
-    image: offen/docker-volume-backup:${TEST_VERSION}
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
     depends_on:
       - minio
     deploy:
@@ -33,6 +33,8 @@ services:
       AWS_S3_BUCKET_NAME: backup
       BACKUP_FILENAME: test.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: 7
+      BACKUP_PRUNING_LEEWAY: 5s
     volumes:
       - pg_data:/backup/pg_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
@@ -41,13 +43,15 @@ services:
     image: offen/offen:latest
     labels:
       - docker-volume-backup.stop-during-backup=true
+    healthcheck:
+      disable: true
     deploy:
       replicas: 2
       restart_policy:
         condition: on-failure
 
   pg:
-    image: postgres:12.2-alpine
+    image: postgres:14-alpine
     environment:
       POSTGRES_PASSWORD: example
     labels:
@@ -60,4 +64,6 @@ services:
 
 volumes:
   backup_data:
+    name: backup_data
   pg_data:
+    name: pg_data

test/swarm/run.sh

@@ -3,13 +3,15 @@
 set -e
 
 cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
 
 docker swarm init
 
 docker stack deploy --compose-file=docker-compose.yml test_stack
 
 while [ -z $(docker ps -q -f name=backup) ]; do
-  echo "[TEST:INFO] Backup container not ready yet. Retrying."
+  info "Backup container not ready yet. Retrying."
   sleep 1
 done
 
@@ -17,20 +19,19 @@ sleep 20
 
 docker exec $(docker ps -q -f name=backup) backup
 
-docker run --rm -it \
-  -v test_stack_backup_data:/data alpine \
+docker run --rm \
+  -v backup_data:/data alpine \
   ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
 
-echo "[TEST:PASS] Found relevant files in untared backup."
+pass "Found relevant files in untared backup."
 
-if [ "$(docker ps -q | wc -l)" != "5" ]; then
-  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
-  docker ps -a
-  exit 1
-fi
-
-echo "[TEST:PASS] All containers running post backup."
+sleep 5
+expect_running_containers "5"
 
 docker stack rm test_stack
-
 docker swarm leave --force
+
+sleep 10
+
+docker volume rm backup_data
+docker volume rm pg_data

test/user/.gitignore

@@ -0,0 +1,2 @@
+local
+backup

test/user/docker-compose.yml

@@ -0,0 +1,30 @@
+version: '2.4'
+
+services:
+  alpine:
+    image: alpine:3.17.3
+    tty: true 
+    volumes:
+      - app_data:/tmp
+    labels:
+      - docker-volume-backup.archive-pre.user=testuser
+      - docker-volume-backup.archive-pre=/bin/sh -c 'whoami > /tmp/whoami.txt'
+
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      EXEC_FORWARD_OUTPUT: "true"
+    volumes:
+      - ./local:/archive
+      - app_data:/backup/data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  app_data:
+  archive:

test/user/run.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker compose up -d
+
+user_name=testuser
+docker exec user-alpine-1 adduser --disabled-password "$user_name"
+
+docker compose exec backup backup
+
+tar -xvf ./local/test.tar.gz
+if [ ! -f ./backup/data/whoami.txt ]; then
+  fail "Could not find file written by pre command."
+fi
+pass "Found expected file."
+
+tar -xvf ./local/test.tar.gz
+if [ "$(cat ./backup/data/whoami.txt)" != "$user_name" ]; then
+  fail "Could not find expected user name."
+fi
+pass "Found expected user."
+
+docker compose down --volumes
+sudo rm -rf ./local
+

test/util.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+
+info () {
+  echo "[test:${current_test:-none}:info] "$1""
+}
+
+pass () {
+  echo "[test:${current_test:-none}:pass] "$1""
+}
+
+fail () {
+  echo "[test:${current_test:-none}:fail] "$1""
+  exit 1
+}
+
+expect_running_containers () {
+  if [ "$(docker ps -q | wc -l)" != "$1" ]; then
+    fail "Expected $1 containers to be running, instead seen: "$(docker ps -a | wc -l)""
+  fi
+  pass "$1 containers running."
+}

test/webdav/docker-compose.yml

@@ -0,0 +1,45 @@
+version: '3'
+
+services:
+  webdav:
+    image: bytemark/webdav:2.4
+    environment:
+      AUTH_TYPE: Digest
+      USERNAME: test
+      PASSWORD: test
+    volumes:
+      - webdav_backup_data:/var/lib/dav
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    depends_on:
+      - webdav
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      WEBDAV_URL: http://webdav/
+      WEBDAV_URL_INSECURE: 'true'
+      WEBDAV_PATH: /my/new/path/
+      WEBDAV_USERNAME: test
+      WEBDAV_PASSWORD: test
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  webdav_backup_data:
+    name: webdav_backup_data
+  app_data:

test/webdav/run.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+docker run --rm \
+  -v webdav_backup_data:/webdav_data \
+  alpine \
+  ash -c 'tar -xvf /webdav_data/data/my/new/path/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in untared remote backup."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+docker compose exec backup backup
+
+docker run --rm \
+  -v webdav_backup_data:/webdav_data \
+  alpine \
+  ash -c '[ $(find /webdav_data/data/my/new/path/ -type f | wc -l) = "1" ]'
+
+pass "Remote backups have not been deleted."
+
+docker compose down --volumes