Add documentation for header format option

.circleci/config.yml

@@ -5,7 +5,6 @@ jobs:
     machine:
       image: ubuntu-2004:202201-02
     working_directory: ~/docker-volume-backup
-    resource_class: large
     steps:
       - checkout
       - run:
@@ -30,7 +29,6 @@ jobs:
           DOCKER_BUILDKIT: '1'
           DOCKER_CLI_EXPERIMENTAL: enabled
     working_directory: ~/docker-volume-backup
-    resource_class: large
     steps:
       - checkout
       - setup_remote_docker:

README.md

@@ -7,7 +7,7 @@
 Backup Docker volumes locally or to any S3 compatible storage.
 
 The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup.
-It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__.
+It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3 or WebDAV compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__.
 
 <!-- MarkdownTOC -->
 
@@ -30,13 +30,11 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
   - [Replace deprecated `BACKUP_FROM_SNAPSHOT` usage](#replace-deprecated-backup_from_snapshot-usage)
   - [Using a custom Docker host](#using-a-custom-docker-host)
   - [Run multiple backup schedules in the same container](#run-multiple-backup-schedules-in-the-same-container)
-  - [Define different retention schedules](#define-different-retention-schedules)
 - [Recipes](#recipes)
   - [Backing up to AWS S3](#backing-up-to-aws-s3)
   - [Backing up to Filebase](#backing-up-to-filebase)
   - [Backing up to MinIO](#backing-up-to-minio)
   - [Backing up to WebDAV](#backing-up-to-webdav)
-  - [Backing up to SSH](#backing-up-to-ssh)
   - [Backing up locally](#backing-up-locally)
   - [Backing up to AWS S3 as well as locally](#backing-up-to-aws-s3-as-well-as-locally)
   - [Running on a custom cron schedule](#running-on-a-custom-cron-schedule)
@@ -169,12 +167,6 @@ You can populate below template according to your requirements and use it as you
 
 # BACKUP_SOURCES="/other/location"
 
-# When given, all files in BACKUP_SOURCES whose full path matches the given
-# regular expression will be excluded from the archive. Regular Expressions
-# can be used as from the Go standard library https://pkg.go.dev/regexp
-
-# BACKUP_EXCLUDE_REGEXP="\.log$"
-
 ########### BACKUP STORAGE
 
 # The name of the remote bucket that should be used for storing backups. If
@@ -215,17 +207,12 @@ You can populate below template according to your requirements and use it as you
 # AWS_ENDPOINT_PROTO="https"
 
 # Setting this variable to `true` will disable verification of
-# SSL certificates for AWS_ENDPOINT. You shouldn't use this unless you use
+# SSL certificates. You shouldn't use this unless you use self-signed
-# self-signed certificates for your remote storage backend. This can only be
+# certificates for your remote storage backend. This can only be used
-# used when AWS_ENDPOINT_PROTO is set to `https`.
+# when AWS_ENDPOINT_PROTO is set to `https`.
 
 # AWS_ENDPOINT_INSECURE="true"
 
-# Setting this variable will change the S3 storage class header.
-# Defaults to "STANDARD", you can set this value according to your needs.
-
-# AWS_STORAGE_CLASS="GLACIER"
-
 # You can also backup files to any WebDAV server:
 
 # The URL of the remote WebDAV server
@@ -245,46 +232,6 @@ You can populate below template according to your requirements and use it as you
 
 # WEBDAV_PASSWORD="password"
 
-# Setting this variable to `true` will disable verification of
-# SSL certificates for WEBDAV_URL. You shouldn't use this unless you use
-# self-signed certificates for your remote storage backend.
-
-# WEBDAV_URL_INSECURE="true"
-
-# You can also backup files to any SSH server:
-
-# The URL of the remote SSH server
-
-# SSH_HOST_NAME="server.local"
-
-# The port of the remote SSH server
-# Optional variable default value is `22`
-
-# SSH_PORT=2222
-
-# The Directory to place the backups to on the SSH server.
-
-# SSH_REMOTE_PATH="/my/directory/"
-
-# The username for the SSH server
-
-# SSH_USER="user"
-
-# The password for the SSH server
-
-# SSH_PASSWORD="password"
-
-# The private key path in container for SSH server
-# Default value: /root/.ssh/id_rsa
-# If file is mounted to /root/.ssh/id_rsa path it will be used. Non-RSA keys will
-# also work.
-
-# SSH_IDENTITY_FILE="/root/.ssh/id_rsa"
-
-# The passphrase for the identity file
-
-# SSH_IDENTITY_PASSPHRASE="pass"
-
 # In addition to storing backups remotely, you can also keep local copies.
 # Pass a container-local path to store your backups if needed. You also need to
 # mount a local folder or Docker volume into that location (`/archive`
@@ -404,6 +351,19 @@ You can populate below template according to your requirements and use it as you
 
 # LOCK_TIMEOUT="60m"
 
+########### HEADER FORMAT USED BY THE TAR ARCHIVE
+
+# By default, tar archive creation will pick a header format that is appropriate
+# for the archive's contents. In case you have special requirements or need to
+# work with tools that do not support all standard header formats, you can use
+# this option to enforce a certain header format. Valid options are "USTAR",
+# "PAX" and "GNU". Be aware that setting this value might create situations where
+# it's not possible to encode the information about a certain file, making the
+# backup fail.
+# In case no value is set, an appropriate format will be selected for each file.
+
+# TAR_ARCHIVE_HEADER_FORMAT="USTAR"
+
 ########### EMAIL NOTIFICATIONS
 
 # ************************************************************************
@@ -435,7 +395,7 @@ You can populate below template according to your requirements and use it as you
 # EMAIL_SMTP_PORT="<port>"
 ```
 
-In case you encouter double quoted values in your configuration you might be running an [older version of `docker-compose`][compose-issue].
+In case you encouter double quoted values in your configuration you might be running an [older version of `docker-compose`].
 You can work around this by either updating `docker-compose` or unquoting your configuration values.
 
 [compose-issue]: https://github.com/docker/compose/issues/2854
@@ -786,39 +746,6 @@ The exact order of schedules that use the same cron expression is not specified.
 In case you need your schedules to overlap, you need to create a dedicated container for each schedule instead.
 When changing the configuration, you currently need to manually restart the container for the changes to take effect.
 
-### Define different retention schedules
-
-If you want to manage backup retention on different schedules, the most straight forward approach is to define a dedicated configuration for retention rule using a different prefix in the `BACKUP_FILENAME` parameter and then run them on different cron schedules.
-
-For example, if you wanted to keep daily backups for 7 days, weekly backups for a month, and retain monthly backups forever, you could create three configuration files and mount them into `/etc/dockervolumebackup.d`:
-
-```ini
-# 01daily.conf
-BACKUP_FILENAME="daily-backup-%Y-%m-%dT%H-%M-%S.tar.gz"
-# run every day at 2am
-BACKUP_CRON_EXPRESSION="0 2 * * *"
-BACKUP_PRUNING_PREFIX="daily-backup-"
-BACKUP_RETENTION_DAYS="7"
-```
-
-```ini
-# 02weekly.conf
-BACKUP_FILENAME="weekly-backup-%Y-%m-%dT%H-%M-%S.tar.gz"
-# run every monday at 3am
-BACKUP_CRON_EXPRESSION="0 3 * * 1"
-BACKUP_PRUNING_PREFIX="weekly-backup-"
-BACKUP_RETENTION_DAYS="31"
-```
-
-```ini
-# 03monthly.conf
-BACKUP_FILENAME="monthly-backup-%Y-%m-%dT%H-%M-%S.tar.gz"
-# run every 1st of a month at 4am
-BACKUP_CRON_EXPRESSION="0 4 1 * *"
-```
-
-Note that while it's possible to define colliding cron schedules for each of these configurations, you might need to adjust the value for `LOCK_TIMEOUT` in case your backups are large and might take longer than an hour.
-
 ## Recipes
 
 This section lists configuration for some real-world use cases that you can mix and match according to your needs.
@@ -910,29 +837,6 @@ volumes:
   data:
 ```
 
-### Backing up to SSH
-
-```yml
-version: '3'
-
-services:
-  # ... define other services using the `data` volume here
-  backup:
-    image: offen/docker-volume-backup:v2
-    environment:
-      SSH_HOST_NAME: server.local
-      SSH_PORT: 2222
-      SSH_USER: user
-      SSH_REMOTE_PATH: /data
-    volumes:
-      - data:/backup/my-app-backup:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - /path/to/private_key:/root/.ssh/id
-
-volumes:
-  data:
-```
-
 ### Backing up locally
 
 ```yml

cmd/backup/archive.go

@@ -11,13 +11,14 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 )
 
-func createArchive(files []string, inputFilePath, outputFilePath string) error {
+func createArchive(inputFilePath, outputFilePath string, options createArchiveOptions) error {
 	inputFilePath = stripTrailingSlashes(inputFilePath)
 	inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
 	if err != nil {
@@ -27,7 +28,7 @@ func createArchive(files []string, inputFilePath, outputFilePath string) error {
 		return fmt.Errorf("createArchive: error creating output file path: %w", err)
 	}
 
-	if err := compress(files, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
+	if err := compress(inputFilePath, outputFilePath, filepath.Dir(inputFilePath), options); err != nil {
 		return fmt.Errorf("createArchive: error creating archive: %w", err)
 	}
 
@@ -51,7 +52,7 @@ func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error)
 	return inputFilePath, outputFilePath, err
 }
 
-func compress(paths []string, outFilePath, subPath string) error {
+func compress(inPath, outFilePath, subPath string, options createArchiveOptions) error {
 	file, err := os.Create(outFilePath)
 	if err != nil {
 		return fmt.Errorf("compress: error creating out file: %w", err)
@@ -61,8 +62,16 @@ func compress(paths []string, outFilePath, subPath string) error {
 	gzipWriter := gzip.NewWriter(file)
 	tarWriter := tar.NewWriter(gzipWriter)
 
+	var paths []string
+	if err := filepath.WalkDir(inPath, func(path string, di fs.DirEntry, err error) error {
+		paths = append(paths, path)
+		return err
+	}); err != nil {
+		return fmt.Errorf("compress: error walking filesystem tree: %w", err)
+	}
+
 	for _, p := range paths {
-		if err := writeTarGz(p, tarWriter, prefix); err != nil {
+		if err := writeTarGz(p, tarWriter, prefix, options.format); err != nil {
 			return fmt.Errorf("compress error writing %s to archive: %w", p, err)
 		}
 	}
@@ -85,7 +94,7 @@ func compress(paths []string, outFilePath, subPath string) error {
 	return nil
 }
 
-func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
+func writeTarGz(path string, tarWriter *tar.Writer, prefix string, format tar.Format) error {
 	fileInfo, err := os.Lstat(path)
 	if err != nil {
 		return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err)
@@ -104,6 +113,10 @@ func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
 	}
 
 	header, err := tar.FileInfoHeader(fileInfo, link)
+	if format >= 0 {
+		header.Format = format
+	}
+
 	if err != nil {
 		return fmt.Errorf("writeTarGz: error getting file info header: %w", err)
 	}
@@ -131,3 +144,7 @@ func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
 
 	return nil
 }
+
+type createArchiveOptions struct {
+	format tar.Format
+}

cmd/backup/config.go

@@ -4,8 +4,8 @@
 package main
 
 import (
+	"archive/tar"
 	"fmt"
-	"regexp"
 	"time"
 )
 
@@ -22,13 +22,11 @@ type Config struct {
 	BackupPruningPrefix        string        `split_words:"true"`
 	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
 	BackupFromSnapshot         bool          `split_words:"true"`
-	BackupExcludeRegexp        RegexpDecoder `split_words:"true"`
 	AwsS3BucketName            string        `split_words:"true"`
 	AwsS3Path                  string        `split_words:"true"`
 	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
 	AwsEndpointProto           string        `split_words:"true" default:"https"`
 	AwsEndpointInsecure        bool          `split_words:"true"`
-	AwsStorageClass            string        `split_words:"true"`
 	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
 	AwsSecretAccessKey         string        `split_words:"true"`
 	AwsIamRoleEndpoint         string        `split_words:"true"`
@@ -42,34 +40,36 @@ type Config struct {
 	EmailSMTPUsername          string        `envconfig:"EMAIL_SMTP_USERNAME"`
 	EmailSMTPPassword          string        `envconfig:"EMAIL_SMTP_PASSWORD"`
 	WebdavUrl                  string        `split_words:"true"`
-	WebdavUrlInsecure          bool          `split_words:"true"`
 	WebdavPath                 string        `split_words:"true" default:"/"`
 	WebdavUsername             string        `split_words:"true"`
 	WebdavPassword             string        `split_words:"true"`
-	SSHHostName                string        `split_words:"true"`
-	SSHPort                    string        `split_words:"true" default:"22"`
-	SSHUser                    string        `split_words:"true"`
-	SSHPassword                string        `split_words:"true"`
-	SSHIdentityFile            string        `split_words:"true" default:"/root/.ssh/id_rsa"`
-	SSHIdentityPassphrase      string        `split_words:"true"`
-	SSHRemotePath              string        `split_words:"true"`
 	ExecLabel                  string        `split_words:"true"`
 	ExecForwardOutput          bool          `split_words:"true"`
 	LockTimeout                time.Duration `split_words:"true" default:"60m"`
+	TarArchiveHeaderFormat     TarFormat     `split_words:"true"`
 }
 
-type RegexpDecoder struct {
+type TarFormat tar.Format
-	Re *regexp.Regexp
-}
 
-func (r *RegexpDecoder) Decode(v string) error {
+func (t *TarFormat) Decode(value string) error {
-	if v == "" {
+	switch value {
+	case "PAX":
+		*t = TarFormat(tar.FormatPAX)
 		return nil
+	case "USTAR":
+		*t = TarFormat(tar.FormatUSTAR)
+		return nil
+	case "GNU":
+		*t = TarFormat(tar.FormatGNU)
+		return nil
+	case "":
+		*t = TarFormat(-1)
+		return nil
+	default:
+		return fmt.Errorf("tarFormat: unknown format %s", value)
 	}
-	re, err := regexp.Compile(v)
+}
-	if err != nil {
+
-		return fmt.Errorf("config: error compiling given regexp `%s`: %w", v, err)
+func (t *TarFormat) Format() tar.Format {
-	}
+	return tar.Format(*t)
-	*r = RegexpDecoder{Re: re}
-	return nil
 }

cmd/backup/notifications.go

@@ -7,7 +7,6 @@ import (
 	"bytes"
 	_ "embed"
 	"fmt"
-	"os"
 	"text/template"
 	"time"
 
@@ -83,7 +82,6 @@ var templateHelpers = template.FuncMap{
 	"formatBytesBin": func(bytes uint64) string {
 		return formatBytes(bytes, false)
 	},
-	"env": os.Getenv,
 }
 
 // formatBytes converts an amount of bytes in a human-readable representation

cmd/backup/script.go

@@ -9,8 +9,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -18,9 +16,6 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/pkg/sftp"
-	"golang.org/x/crypto/ssh"
-
 	"github.com/containrrr/shoutrrr"
 	"github.com/containrrr/shoutrrr/pkg/router"
 	"github.com/docker/docker/api/types"
@@ -43,8 +38,6 @@ type script struct {
 	cli          *client.Client
 	minioClient  *minio.Client
 	webdavClient *gowebdav.Client
-	sshClient    *ssh.Client
-	sftpClient   *sftp.Client
 	logger       *logrus.Logger
 	sender       *router.ServiceRouter
 	template     *template.Template
@@ -153,66 +146,6 @@ func newScript() (*script, error) {
 		} else {
 			webdavClient := gowebdav.NewClient(s.c.WebdavUrl, s.c.WebdavUsername, s.c.WebdavPassword)
 			s.webdavClient = webdavClient
-			if s.c.WebdavUrlInsecure {
-				defaultTransport, ok := http.DefaultTransport.(*http.Transport)
-				if !ok {
-					return nil, errors.New("newScript: unexpected error when asserting type for http.DefaultTransport")
-				}
-				webdavTransport := defaultTransport.Clone()
-				webdavTransport.TLSClientConfig.InsecureSkipVerify = s.c.WebdavUrlInsecure
-				s.webdavClient.SetTransport(webdavTransport)
-			}
-		}
-	}
-
-	if s.c.SSHHostName != "" {
-		var authMethods []ssh.AuthMethod
-
-		if s.c.SSHPassword != "" {
-			authMethods = append(authMethods, ssh.Password(s.c.SSHPassword))
-		}
-
-		if _, err := os.Stat(s.c.SSHIdentityFile); err == nil {
-			key, err := ioutil.ReadFile(s.c.SSHIdentityFile)
-			if err != nil {
-				return nil, errors.New("newScript: error reading the private key")
-			}
-
-			var signer ssh.Signer
-			if s.c.SSHIdentityPassphrase != "" {
-				signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(s.c.SSHIdentityPassphrase))
-				if err != nil {
-					return nil, errors.New("newScript: error parsing the encrypted private key")
-				}
-				authMethods = append(authMethods, ssh.PublicKeys(signer))
-			} else {
-				signer, err = ssh.ParsePrivateKey(key)
-				if err != nil {
-					return nil, errors.New("newScript: error parsing the private key")
-				}
-				authMethods = append(authMethods, ssh.PublicKeys(signer))
-			}
-		}
-
-		sshClientConfig := &ssh.ClientConfig{
-			User:            s.c.SSHUser,
-			Auth:            authMethods,
-			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		}
-		sshClient, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", s.c.SSHHostName, s.c.SSHPort), sshClientConfig)
-		s.sshClient = sshClient
-		if err != nil {
-			return nil, fmt.Errorf("newScript: error creating ssh client: %w", err)
-		}
-		_, _, err = s.sshClient.SendRequest("keepalive", false, nil)
-		if err != nil {
-			return nil, err
-		}
-
-		sftpClient, err := sftp.NewClient(sshClient)
-		s.sftpClient = sftpClient
-		if err != nil {
-			return nil, fmt.Errorf("newScript: error creating sftp client: %w", err)
 		}
 	}
 
@@ -455,28 +388,9 @@ func (s *script) takeBackup() error {
 		s.logger.Infof("Removed tar file `%s`.", tarFile)
 		return nil
 	})
-
+	if err := createArchive(backupSources, tarFile, createArchiveOptions{
-	backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources))
+		format: s.c.TarArchiveHeaderFormat.Format(),
-	if err != nil {
-		return fmt.Errorf("takeBackup: error getting absolute path: %w", err)
-	}
-
-	var filesEligibleForBackup []string
-	if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) {
-			return nil
-		}
-		filesEligibleForBackup = append(filesEligibleForBackup, path)
-		return nil
 	}); err != nil {
-		return fmt.Errorf("compress: error walking filesystem tree: %w", err)
-	}
-
-	if err := createArchive(filesEligibleForBackup, backupSources, tarFile); err != nil {
 		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
 	}
 
@@ -548,8 +462,7 @@ func (s *script) copyBackup() error {
 
 	if s.minioClient != nil {
 		if _, err := s.minioClient.FPutObject(context.Background(), s.c.AwsS3BucketName, filepath.Join(s.c.AwsS3Path, name), s.file, minio.PutObjectOptions{
-			ContentType:  "application/tar+gzip",
+			ContentType: "application/tar+gzip",
-			StorageClass: s.c.AwsStorageClass,
 		}); err != nil {
 			return fmt.Errorf("copyBackup: error uploading backup to remote storage: %w", err)
 		}
@@ -570,52 +483,6 @@ func (s *script) copyBackup() error {
 		s.logger.Infof("Uploaded a copy of backup `%s` to WebDAV-URL '%s' at path '%s'.", s.file, s.c.WebdavUrl, s.c.WebdavPath)
 	}
 
-	if s.sshClient != nil {
-		source, err := os.Open(s.file)
-		if err != nil {
-			return fmt.Errorf("copyBackup: error reading the file to be uploaded: %w", err)
-		}
-		defer source.Close()
-
-		destination, err := s.sftpClient.Create(filepath.Join(s.c.SSHRemotePath, name))
-		if err != nil {
-			return fmt.Errorf("copyBackup: error creating file on SSH storage: %w", err)
-		}
-		defer destination.Close()
-
-		chunk := make([]byte, 1000000)
-		for {
-			num, err := source.Read(chunk)
-			if err == io.EOF {
-				tot, err := destination.Write(chunk[:num])
-				if err != nil {
-					return fmt.Errorf("copyBackup: error uploading the file to SSH storage: %w", err)
-				}
-
-				if tot != len(chunk[:num]) {
-					return fmt.Errorf("sshClient: failed to write stream")
-				}
-
-				break
-			}
-
-			if err != nil {
-				return fmt.Errorf("copyBackup: error uploading the file to SSH storage: %w", err)
-			}
-
-			tot, err := destination.Write(chunk[:num])
-			if err != nil {
-				return fmt.Errorf("copyBackup: error uploading the file to SSH storage: %w", err)
-			}
-
-			if tot != len(chunk[:num]) {
-				return fmt.Errorf("sshClient: failed to write stream")
-			}
-		}
-
-		s.logger.Infof("Uploaded a copy of backup `%s` to SSH storage '%s' at path '%s'.", s.file, s.c.SSHHostName, s.c.SSHRemotePath)
-	}
-
 	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
 		if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
 			return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
@@ -749,37 +616,6 @@ func (s *script) pruneBackups() error {
 		})
 	}
 
-	if s.sshClient != nil {
-		candidates, err := s.sftpClient.ReadDir(s.c.SSHRemotePath)
-		if err != nil {
-			return fmt.Errorf("pruneBackups: error reading directory from SSH storage: %w", err)
-		}
-
-		var matches []string
-		for _, candidate := range candidates {
-			if !strings.HasPrefix(candidate.Name(), s.c.BackupPruningPrefix) {
-				continue
-			}
-			if candidate.ModTime().Before(deadline) {
-				matches = append(matches, candidate.Name())
-			}
-		}
-
-		s.stats.Storages.SSH = StorageStats{
-			Total:  uint(len(candidates)),
-			Pruned: uint(len(matches)),
-		}
-
-		doPrune(len(matches), len(candidates), "SSH backup(s)", func() error {
-			for _, match := range matches {
-				if err := s.sftpClient.Remove(filepath.Join(s.c.SSHRemotePath, match)); err != nil {
-					return fmt.Errorf("pruneBackups: error removing file from SSH storage: %w", err)
-				}
-			}
-			return nil
-		})
-	}
-
 	if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
 		globPattern := path.Join(
 			s.c.BackupArchive,

cmd/backup/stats.go

@@ -30,11 +30,10 @@ type StorageStats struct {
 	PruneErrors uint
 }
 
-// StoragesStats stats about each possible archival location (Local, WebDAV, SSH, S3)
+// StoragesStats stats about each possible archival location (Local, WebDAV, S3)
 type StoragesStats struct {
 	Local  StorageStats
 	WebDAV StorageStats
-	SSH    StorageStats
 	S3     StorageStats
 }
 

docs/NOTIFICATION-TEMPLATES.md

@@ -25,16 +25,15 @@ Here is a list of all data passed to the template:
     * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
     * `Size`: size in bytes of the backup file
   * `Storages`: object that holds stats about each storage
-    * `Local`, `S3`, `WebDAV` or `SSH`:
+    * `Local`, `S3` or `WebDAV`:
       * `Total`: total number of backup files
       * `Pruned`: number of backup files that were deleted due to pruning rule
       * `PruneErrors`: number of backup files that were unable to be pruned
 
 ## Functions
 
-Some formatting and helper functions are also available:
+Some formatting functions are also available:
 
 * `formatTime`: formats a time object using [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format (e.g. `2022-02-11T01:00:00Z`)
 * `formatBytesBin`: formats an amount of bytes using powers of 1024 (e.g. `7055258` bytes will be `6.7 MiB`) 
 * `formatBytesDec`: formats an amount of bytes using powers of 1000 (e.g. `7055258` bytes will be `7.1 MB`)
-* `env`: returns the value of the environment variable of the given key if set

entrypoint.sh

@@ -22,4 +22,4 @@ else
 fi
 
 echo "Starting cron in foreground."
-crond -f -d 8
+crond -f -l 8

go.mod

@@ -11,16 +11,15 @@ require (
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
 	github.com/minio/minio-go/v7 v7.0.16
 	github.com/otiai10/copy v1.7.0
-	github.com/pkg/sftp v1.13.5
 	github.com/sirupsen/logrus v1.8.1
-	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
+	github.com/studio-b12/gowebdav v0.0.0-20211109083228-3f8721cd4b6f
-	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
-	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 )
 
 require (
-	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/Microsoft/go-winio v0.4.17 // indirect
-	github.com/containerd/containerd v1.6.6 // indirect
+	github.com/containerd/containerd v1.5.5 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
@@ -28,39 +27,33 @@ require (
 	github.com/fatih/color v1.10.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.6 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
-	github.com/kr/fs v0.1.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/moby/term v0.0.0-20200312100748-672ec06f55cd // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/nxadm/tail v1.4.6 // indirect
 	github.com/onsi/ginkgo v1.14.2 // indirect
 	github.com/onsi/gomega v1.10.3 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rs/xid v1.3.0 // indirect
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect
+	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
-	google.golang.org/grpc v1.47.0 // indirect
+	google.golang.org/grpc v1.33.2 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/ini.v1 v1.65.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

test/cli/run.sh

@@ -3,8 +3,6 @@
 set -e
 
 cd $(dirname $0)
-. ../util.sh
-current_test=$(basename $(pwd))
 
 docker network create test_network
 docker volume create backup_data
@@ -52,11 +50,17 @@ docker run --rm -it \
   -v backup_data:/data alpine \
   ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db && test -d /backup/empty_data'
 
-pass "Found relevant files in untared remote backup."
+echo "[TEST:PASS] Found relevant files in untared remote backup."
 
 # This test does not stop containers during backup. This is happening on
 # purpose in order to cover this setup as well.
-expect_running_containers "2"
+if [ "$(docker ps -q | wc -l)" != "2" ]; then
+  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
+  docker ps
+  exit 1
+fi
+
+echo "[TEST:PASS] All containers running post backup."
 
 docker rm $(docker stop minio offen)
 docker volume rm backup_data app_data

test/commands/run.sh

@@ -3,8 +3,7 @@
 set -e
 
 cd $(dirname $0)
-. ../util.sh
+
-current_test=$(basename $(pwd))
 
 docker-compose up -d
 sleep 30 # mariadb likes to take a bit before responding
@@ -14,27 +13,29 @@ sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' commands_archive
 
 tar -xvf ./local/test.tar.gz
 if [ ! -f ./backup/data/dump.sql ]; then
-  fail "Could not find file written by pre command."
+  echo "[TEST:FAIL] Could not find file written by pre command."
+  exit 1
 fi
-pass "Found expected file."
+echo "[TEST:PASS] Found expected file."
 
 if [ -f ./backup/data/post.txt ]; then
-  fail "File created in post command was present in backup."
+  echo "[TEST:FAIL] File created in post command was present in backup."
+  exit 1
 fi
-pass "Did not find unexpected file."
+echo "[TEST:PASS] Did not find unexpected file."
 
 docker-compose down --volumes
 sudo rm -rf ./local
 
 
-info "Running commands test in swarm mode next."
+echo "[TEST:INFO] Running commands test in swarm mode next."
 
 docker swarm init
 
 docker stack deploy --compose-file=docker-compose.yml test_stack
 
 while [ -z $(docker ps -q -f name=backup) ]; do
-  info "Backup container not ready yet. Retrying."
+  echo "[TEST:INFO] Backup container not ready yet. Retrying."
   sleep 1
 done
 
@@ -46,14 +47,16 @@ sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' test_stack_archi
 
 tar -xvf ./local/test.tar.gz
 if [ ! -f ./backup/data/dump.sql ]; then
-  fail "Could not find file written by pre command."
+  echo "[TEST:FAIL] Could not find file written by pre command."
+  exit 1
 fi
-pass "Found expected file."
+echo "[TEST:PASS] Found expected file."
 
 if [ -f ./backup/data/post.txt ]; then
-  fail "File created in post command was present in backup."
+  echo "[TEST:FAIL] File created in post command was present in backup."
+  exit 1
 fi
-pass "Did not find unexpected file."
+echo "[TEST:PASS] Did not find unexpected file."
 
 docker stack rm test_stack
 docker swarm leave --force

test/compose/docker-compose.yml

@@ -12,11 +12,21 @@ services:
     volumes:
       - minio_backup_data:/data
 
+  webdav:
+    image: bytemark/webdav:2.4
+    environment:
+      AUTH_TYPE: Digest
+      USERNAME: test
+      PASSWORD: test
+    volumes:
+      - webdav_backup_data:/var/lib/dav
+
   backup:
     image: offen/docker-volume-backup:${TEST_VERSION:-canary}
     hostname: hostnametoken
     depends_on:
       - minio
+      - webdav
     restart: always
     environment:
       AWS_ACCESS_KEY_ID: test
@@ -26,11 +36,18 @@ services:
       AWS_S3_BUCKET_NAME: backup
       BACKUP_FILENAME_EXPAND: 'true'
       BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
+      GPG_PASSPHRASE: 1234secret
+      WEBDAV_URL: http://webdav/
+      WEBDAV_PATH: /my/new/path/
+      WEBDAV_USERNAME: test
+      WEBDAV_PASSWORD: test
     volumes:
+      - ./local:/archive
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 
@@ -43,5 +60,5 @@ services:
 
 volumes:
   minio_backup_data:
-    name: minio_backup_data
+  webdav_backup_data:
   app_data:

test/compose/run.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+
+mkdir -p local
+
+docker-compose up -d
+sleep 5
+
+# A symlink for a known file in the volume is created so the test can check
+# whether symlinks are preserved on backup.
+docker-compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link
+docker-compose exec backup backup
+
+sleep 5
+if [ "$(docker-compose ps -q | wc -l)" != "4" ]; then
+  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
+  docker-compose ps
+  exit 1
+fi
+echo "[TEST:PASS] All containers running post backup."
+
+
+docker run --rm -it \
+  -v compose_minio_backup_data:/minio_data \
+  -v compose_webdav_backup_data:/webdav_data alpine \
+  ash -c 'apk add gnupg && \
+          echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /minio_data/backup/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xvf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db && \
+          echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /webdav_data/data/my/new/path/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xvf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+echo "[TEST:PASS] Found relevant files in decrypted and untared remote backups."
+
+echo 1234secret | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 ./local/test-hostnametoken.tar.gz.gpg > ./local/decrypted.tar.gz
+tar -xf ./local/decrypted.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
+rm ./local/decrypted.tar.gz
+test -L /tmp/backup/app_data/db.link
+
+echo "[TEST:PASS] Found relevant files in decrypted and untared local backup."
+
+test -L ./local/test-hostnametoken.latest.tar.gz.gpg
+echo "[TEST:PASS] Found symlink to latest version in local backup."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker-compose up -d
+sleep 5
+
+docker-compose exec backup backup
+
+docker run --rm -it \
+  -v compose_minio_backup_data:/minio_data \
+  -v compose_webdav_backup_data:/webdav_data alpine \
+  ash -c '[ $(find /minio_data/backup/ -type f | wc -l) = "1" ] && \
+          [ $(find /webdav_data/data/my/new/path/ -type f | wc -l) = "1" ]'
+
+echo "[TEST:PASS] Remote backups have not been deleted."
+
+if [ "$(find ./local -type f | wc -l)" != "1" ]; then
+  echo "[TEST:FAIL] Backups should not have been deleted, instead seen:"
+  find ./local -type f
+  exit 1
+fi
+echo "[TEST:PASS] Local backups have not been deleted."
+
+docker-compose down --volumes

test/confd/run.sh

@@ -3,8 +3,6 @@
 set -e
 
 cd $(dirname $0)
-. ../util.sh
-current_test=$(basename $(pwd))
 
 mkdir -p local
 
@@ -16,16 +14,19 @@ sleep 100
 docker-compose down --volumes
 
 if [ ! -f ./local/conf.tar.gz ]; then
-  fail "Config from file was not used."
+  echo "[TEST:FAIL] Config from file was not used."
+  exit 1
 fi
-pass "Config from file was used."
+echo "[TEST:PASS] Config from file was used."
 
 if [ ! -f ./local/other.tar.gz ]; then
-  fail "Run on same schedule did not succeed."
+  echo "[TEST:FAIL] Run on same schedule did not succeed."
+  exit 1
 fi
-pass "Run on same schedule succeeded."
+echo "[TEST:PASS] Run on same schedule succeeded."
 
 if [ -f ./local/never.tar.gz ]; then
-  fail "Unexpected file was found."
+  echo "[TEST:FAIL] Unexpected file was found."
+  exit 1
 fi
-pass "Unexpected cron did not run."
+echo "[TEST:PASS] Unexpected cron did not run."

test/gpg/docker-compose.yml

@@ -1,26 +0,0 @@
-version: '3'
-
-services:
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    restart: always
-    environment:
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_FILENAME: test.tar.gz
-      BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
-      GPG_PASSPHRASE: 1234secret
-    volumes:
-      - ./local:/archive
-      - app_data:/backup/app_data:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-
-  offen:
-    image: offen/offen:latest
-    labels:
-      - docker-volume-backup.stop-during-backup=true
-    volumes:
-      - app_data:/var/opt/offen
-
-volumes:
-  app_data:

test/gpg/run.sh

@@ -1,34 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd "$(dirname "$0")"
-. ../util.sh
-current_test=$(basename $(pwd))
-
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-expect_running_containers "2"
-
-tmp_dir=$(mktemp -d)
-
-echo 1234secret | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 ./local/test.tar.gz.gpg > ./local/decrypted.tar.gz
-tar -xf ./local/decrypted.tar.gz -C $tmp_dir
-ls -lah $tmp_dir
-if [ ! -f $tmp_dir/backup/app_data/offen.db ]; then
-  fail "Could not find expected file in untared archive."
-fi
-rm ./local/decrypted.tar.gz
-
-pass "Found relevant files in decrypted and untared local backup."
-
-if [ ! -L ./local/test-latest.tar.gz.gpg ]; then
-  fail "Could not find local symlink to latest encrypted backup."
-fi
-
-docker-compose down --volumes

test/ignore/.gitignore

@@ -1 +0,0 @@
-local

test/ignore/docker-compose.yml

@@ -1,15 +0,0 @@
-version: '3.8'
-
-services:
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    deploy:
-      restart_policy:
-        condition: on-failure
-    environment:
-      BACKUP_FILENAME: test.tar.gz
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_EXCLUDE_REGEXP: '\.(me|you)$$'
-    volumes:
-      - ./local:/archive
-      - ./sources:/backup/data:ro

test/ignore/run.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(dirname $0)
-. ../util.sh
-current_test=$(basename $(pwd))
-
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-docker-compose exec backup backup
-
-docker-compose down --volumes
-
-out=$(mktemp -d)
-sudo tar --same-owner -xvf ./local/test.tar.gz -C "$out"
-
-if [ ! -f "$out/backup/data/me.txt" ]; then
-  fail "Expected file was not found."
-fi
-pass "Expected file was found."
-
-if [ -f "$out/backup/data/skip.me" ]; then
-  fail "Ignored file was found."
-fi
-pass "Ignored file was not found."

test/local/.gitignore

@@ -1 +0,0 @@
-local

test/local/docker-compose.yml

@@ -1,29 +0,0 @@
-version: '3'
-
-services:
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    hostname: hostnametoken
-    restart: always
-    environment:
-      BACKUP_FILENAME_EXPAND: 'true'
-      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
-      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
-      BACKUP_PRUNING_LEEWAY: 5s
-      BACKUP_PRUNING_PREFIX: test
-    volumes:
-      - app_data:/backup/app_data:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-      - ./local:/archive
-
-  offen:
-    image: offen/offen:latest
-    labels:
-      - docker-volume-backup.stop-during-backup=true
-    volumes:
-      - app_data:/var/opt/offen
-
-volumes:
-  app_data:

test/local/run.sh

@@ -1,55 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd "$(dirname "$0")"
-. ../util.sh
-current_test=$(basename $(pwd))
-
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-
-# A symlink for a known file in the volume is created so the test can check
-# whether symlinks are preserved on backup.
-docker-compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link
-docker-compose exec backup backup
-
-sleep 5
-
-expect_running_containers "2"
-
-tmp_dir=$(mktemp -d)
-tar -xvf ./local/test-hostnametoken.tar.gz -C $tmp_dir
-if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then
-  fail "Could not find expected file in untared archive."
-fi
-rm -f ./local/test-hostnametoken.tar.gz
-
-if [ ! -L "$tmp_dir/backup/app_data/db.link" ]; then
-  fail "Could not find expected symlink in untared archive."
-fi
-
-pass "Found relevant files in decrypted and untared local backup."
-
-if [ ! -L ./local/test-hostnametoken.latest.tar.gz.gpg ]; then
-  fail "Could not find symlink to latest version."
-fi
-
-pass "Found symlink to latest version in local backup."
-
-# The second part of this test checks if backups get deleted when the retention
-# is set to 0 days (which it should not as it would mean all backups get deleted)
-# TODO: find out if we can test actual deletion without having to wait for a day
-BACKUP_RETENTION_DAYS="0" docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-if [ "$(find ./local -type f | wc -l)" != "1" ]; then
-  fail "Backups should not have been deleted, instead seen: "$(find ./local -type f)""
-fi
-pass "Local backups have not been deleted."
-
-docker-compose down --volumes

test/notifications/docker-compose.yml

@@ -10,7 +10,6 @@ services:
       BACKUP_PRUNING_PREFIX: test
       NOTIFICATION_LEVEL: info
       NOTIFICATION_URLS: ${NOTIFICATION_URLS}
-      EXTRA_VALUE: extra-value
     volumes:
       - ./local:/archive
       - app_data:/backup/app_data:ro

test/notifications/notifications.tmpl

@@ -1,5 +1,5 @@
 {{ define "title_success" -}}
-Successful test run with {{ env "EXTRA_VALUE" }}, yay!
+Successful test run, yay!
 {{- end }}
 
 {{ define "body_success" -}}

test/notifications/run.sh

@@ -3,8 +3,6 @@
 set -e
 
 cd $(dirname $0)
-. ../util.sh
-current_test=$(basename $(pwd))
 
 mkdir -p local
 
@@ -12,15 +10,16 @@ docker-compose up -d
 sleep 5
 
 GOTIFY_TOKEN=$(curl -sSLX POST -H 'Content-Type: application/json' -d '{"name":"test"}' http://admin:custom@localhost:8080/application | jq -r '.token')
-info "Set up Gotify application using token $GOTIFY_TOKEN"
+echo "[TEST:INFO] Set up Gotify application using token $GOTIFY_TOKEN"
 
 docker-compose exec backup backup
 
 NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
 if [ "$NUM_MESSAGES" != 0 ]; then
-  fail "Expected no notifications to be sent when not configured"
+  echo "[TEST:FAIL] Expected no notifications to be sent when not configured"
+  exit 1
 fi
-pass "No notifications were sent when not configured."
+echo "[TEST:PASS] No notifications were sent when not configured."
 
 docker-compose down
 
@@ -30,21 +29,24 @@ docker-compose exec backup backup
 
 NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length')
 if [ "$NUM_MESSAGES" != 1 ]; then
-  fail "Expected one notifications to be sent when configured"
+  echo "[TEST:FAIL] Expected one notifications to be sent when configured"
+  exit 1
 fi
-pass "Correct number of notifications were sent when configured."
+echo "[TEST:PASS] Correct number of notifications were sent when configured."
 
 MESSAGE_TITLE=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].title')
 MESSAGE_BODY=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].message')
 
-if [ "$MESSAGE_TITLE" != "Successful test run with extra-value, yay!" ]; then
+if [ "$MESSAGE_TITLE" != "Successful test run, yay!" ]; then
-  fail "Unexpected notification title $MESSAGE_TITLE"
+  echo "[TEST:FAIL] Unexpected notification title $MESSAGE_TITLE"
+  exit 1
 fi
-pass "Custom notification title was used."
+echo "[TEST:PASS] Custom notification title was used."
 
 if [ "$MESSAGE_BODY" != "Backing up /tmp/test.tar.gz succeeded." ]; then
-  fail "Unexpected notification body $MESSAGE_BODY"
+  echo "[TEST:FAIL] Unexpected notification body $MESSAGE_BODY"
+  exit 1
 fi
-pass "Custom notification body was used."
+echo "[TEST:PASS] Custom notification body was used."
 
 docker-compose down --volumes

test/ownership/run.sh

@@ -4,8 +4,6 @@
 set -e
 
 cd $(dirname $0)
-. ../util.sh
-current_test=$(basename $(pwd))
 
 mkdir -p local
 
@@ -14,17 +12,17 @@ sleep 5
 
 docker-compose exec backup backup
 
-tmp_dir=$(mktemp -d)
+sudo tar --same-owner -xvf ./local/backup.tar.gz -C /tmp
-sudo tar --same-owner -xvf ./local/backup.tar.gz -C $tmp_dir
 
-sudo find $tmp_dir/backup/postgres > /dev/null
+sudo find /tmp/backup/postgres > /dev/null
-pass "Backup contains files at expected location"
+echo "[TEST:PASS] Backup contains files at expected location"
 
-for file in $(sudo find $tmp_dir/backup/postgres); do
+for file in $(sudo find /tmp/backup/postgres); do
   if [ "$(sudo stat -c '%u:%g' $file)" != "70:70" ]; then
-    fail "Unexpected file ownership for $file: $(sudo stat -c '%u:%g' $file)"
+    echo "[TEST:FAIL] Unexpected file ownership for $file: $(sudo stat -c '%u:%g' $file)"
+    exit 1
   fi
 done
-pass "All files and directories in backup preserved their ownership."
+echo "[TEST:PASS] All files and directories in backup preserved their ownership."
 
 docker-compose down --volumes

test/s3/run.sh

@@ -1,42 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd "$(dirname "$0")"
-. ../util.sh
-current_test=$(basename $(pwd))
-
-docker-compose up -d
-sleep 5
-
-# A symlink for a known file in the volume is created so the test can check
-# whether symlinks are preserved on backup.
-docker-compose exec backup backup
-
-sleep 5
-
-expect_running_containers "3"
-
-docker run --rm -it \
-  -v minio_backup_data:/minio_data \
-  alpine \
-  ash -c 'tar -xvf /minio_data/backup/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
-
-pass "Found relevant files in untared remote backups."
-
-# The second part of this test checks if backups get deleted when the retention
-# is set to 0 days (which it should not as it would mean all backups get deleted)
-# TODO: find out if we can test actual deletion without having to wait for a day
-BACKUP_RETENTION_DAYS="0" docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-docker run --rm -it \
-  -v minio_backup_data:/minio_data \
-  alpine \
-  ash -c '[ $(find /minio_data/backup/ -type f | wc -l) = "1" ]'
-
-pass "Remote backups have not been deleted."
-
-docker-compose down --volumes

test/ssh/docker-compose.yml

@@ -1,47 +0,0 @@
-version: '3'
-
-services:
-  ssh:
-    image: linuxserver/openssh-server:version-8.6_p1-r3
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - USER_NAME=test
-    volumes:
-      - ./id_rsa.pub:/config/.ssh/authorized_keys
-      - ssh_backup_data:/tmp
-
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    hostname: hostnametoken
-    depends_on:
-      - ssh
-    restart: always
-    environment:
-      BACKUP_FILENAME_EXPAND: 'true'
-      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
-      BACKUP_PRUNING_LEEWAY: 5s
-      BACKUP_PRUNING_PREFIX: test
-      SSH_HOST_NAME: ssh
-      SSH_PORT: 2222
-      SSH_USER: test
-      SSH_REMOTE_PATH: /tmp
-      SSH_IDENTITY_PASSPHRASE: test1234
-    volumes:
-      - ./id_rsa:/root/.ssh/id_rsa
-      - app_data:/backup/app_data:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-
-  offen:
-    image: offen/offen:latest
-    labels:
-      - docker-volume-backup.stop-during-backup=true
-    volumes:
-      - app_data:/var/opt/offen
-
-volumes:
-  ssh_backup_data:
-    name: ssh_backup_data
-  app_data:

test/ssh/run.sh

@@ -1,43 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd "$(dirname "$0")"
-. ../util.sh
-current_test=$(basename $(pwd))
-
-ssh-keygen -t rsa -m pem -b 4096 -N "test1234" -f id_rsa -C "docker-volume-backup@local"
-
-docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-sleep 5
-
-expect_running_containers 3
-
-docker run --rm -it \
-  -v ssh_backup_data:/ssh_data \
-  alpine \
-  ash -c 'tar -xvf /ssh_data/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
-
-pass "Found relevant files in decrypted and untared remote backups."
-
-# The second part of this test checks if backups get deleted when the retention
-# is set to 0 days (which it should not as it would mean all backups get deleted)
-# TODO: find out if we can test actual deletion without having to wait for a day
-BACKUP_RETENTION_DAYS="0" docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-docker run --rm -it \
-  -v ssh_backup_data:/ssh_data \
-  alpine \
-  ash -c '[ $(find /ssh_data/ -type f | wc -l) = "1" ]'
-
-pass "Remote backups have not been deleted."
-
-docker-compose down --volumes
-rm -f id_rsa id_rsa.pub

test/swarm/docker-compose.yml

@@ -64,5 +64,4 @@ services:
 
 volumes:
   backup_data:
-    name: backup_data
   pg_data:

test/swarm/run.sh

@@ -3,15 +3,13 @@
 set -e
 
 cd $(dirname $0)
-. ../util.sh
-current_test=$(basename $(pwd))
 
 docker swarm init
 
 docker stack deploy --compose-file=docker-compose.yml test_stack
 
 while [ -z $(docker ps -q -f name=backup) ]; do
-  info "Backup container not ready yet. Retrying."
+  echo "[TEST:INFO] Backup container not ready yet. Retrying."
   sleep 1
 done
 
@@ -20,13 +18,18 @@ sleep 20
 docker exec $(docker ps -q -f name=backup) backup
 
 docker run --rm -it \
-  -v backup_data:/data alpine \
+  -v test_stack_backup_data:/data alpine \
   ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
 
-pass "Found relevant files in untared backup."
+echo "[TEST:PASS] Found relevant files in untared backup."
 
 sleep 5
-expect_running_containers "5"
+if [ "$(docker ps -q | wc -l)" != "5" ]; then
+  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
+  docker ps -a
+  exit 1
+fi
+echo "[TEST:PASS] All containers running post backup."
 
 docker stack rm test_stack
 docker swarm leave --force

test/util.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-set -e
-
-info () {
-  echo "[test:${current_test:-none}:info] "$1""
-}
-
-pass () {
-  echo "[test:${current_test:-none}:pass] "$1""
-}
-
-fail () {
-  echo "[test:${current_test:-none}:fail] "$1""
-  exit 1
-}
-
-expect_running_containers () {
-  if [ "$(docker ps -q | wc -l)" != "$1" ]; then
-    fail "Expected $1 containers to be running, instead seen: "$(docker ps -a | wc -l)""
-  fi
-  pass "$1 containers running."
-}

test/webdav/docker-compose.yml

@@ -1,45 +0,0 @@
-version: '3'
-
-services:
-  webdav:
-    image: bytemark/webdav:2.4
-    environment:
-      AUTH_TYPE: Digest
-      USERNAME: test
-      PASSWORD: test
-    volumes:
-      - webdav_backup_data:/var/lib/dav
-
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    hostname: hostnametoken
-    depends_on:
-      - webdav
-    restart: always
-    environment:
-      BACKUP_FILENAME_EXPAND: 'true'
-      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
-      BACKUP_PRUNING_LEEWAY: 5s
-      BACKUP_PRUNING_PREFIX: test
-      WEBDAV_URL: http://webdav/
-      WEBDAV_URL_INSECURE: 'true'
-      WEBDAV_PATH: /my/new/path/
-      WEBDAV_USERNAME: test
-      WEBDAV_PASSWORD: test
-    volumes:
-      - app_data:/backup/app_data:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-
-  offen:
-    image: offen/offen:latest
-    labels:
-      - docker-volume-backup.stop-during-backup=true
-    volumes:
-      - app_data:/var/opt/offen
-
-volumes:
-  webdav_backup_data:
-    name: webdav_backup_data
-  app_data:

test/webdav/run.sh

@@ -1,40 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd "$(dirname "$0")"
-. ../util.sh
-current_test=$(basename $(pwd))
-
-docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-sleep 5
-
-expect_running_containers "3"
-
-docker run --rm -it \
-  -v webdav_backup_data:/webdav_data \
-  alpine \
-  ash -c 'tar -xvf /webdav_data/data/my/new/path/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
-
-pass "Found relevant files in untared remote backup."
-
-# The second part of this test checks if backups get deleted when the retention
-# is set to 0 days (which it should not as it would mean all backups get deleted)
-# TODO: find out if we can test actual deletion without having to wait for a day
-BACKUP_RETENTION_DAYS="0" docker-compose up -d
-sleep 5
-
-docker-compose exec backup backup
-
-docker run --rm -it \
-  -v webdav_backup_data:/webdav_data \
-  alpine \
-  ash -c '[ $(find /webdav_data/data/my/new/path/ -type f | wc -l) = "1" ]'
-
-pass "Remote backups have not been deleted."
-
-docker-compose down --volumes