Add documentation for header format option

README.md

@@ -30,7 +30,6 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
   - [Replace deprecated `BACKUP_FROM_SNAPSHOT` usage](#replace-deprecated-backup_from_snapshot-usage)
   - [Using a custom Docker host](#using-a-custom-docker-host)
   - [Run multiple backup schedules in the same container](#run-multiple-backup-schedules-in-the-same-container)
-  - [Define different retention schedules](#define-different-retention-schedules)
 - [Recipes](#recipes)
   - [Backing up to AWS S3](#backing-up-to-aws-s3)
   - [Backing up to Filebase](#backing-up-to-filebase)
@@ -168,12 +167,6 @@ You can populate below template according to your requirements and use it as you
 
 # BACKUP_SOURCES="/other/location"
 
-# When given, all files in BACKUP_SOURCES whose full path matches the given
-# regular expression will be excluded from the archive. Regular Expressions
-# can be used as from the Go standard library https://pkg.go.dev/regexp
-
-# BACKUP_EXCLUDE_REGEXP="\.log$"
-
 ########### BACKUP STORAGE
 
 # The name of the remote bucket that should be used for storing backups. If
@@ -214,9 +207,9 @@ You can populate below template according to your requirements and use it as you
 # AWS_ENDPOINT_PROTO="https"
 
 # Setting this variable to `true` will disable verification of
-# SSL certificates for AWS_ENDPOINT. You shouldn't use this unless you use
-# self-signed certificates for your remote storage backend. This can only be
-# used when AWS_ENDPOINT_PROTO is set to `https`.
+# SSL certificates. You shouldn't use this unless you use self-signed
+# certificates for your remote storage backend. This can only be used
+# when AWS_ENDPOINT_PROTO is set to `https`.
 
 # AWS_ENDPOINT_INSECURE="true"
 
@@ -239,12 +232,6 @@ You can populate below template according to your requirements and use it as you
 
 # WEBDAV_PASSWORD="password"
 
-# Setting this variable to `true` will disable verification of
-# SSL certificates for WEBDAV_URL. You shouldn't use this unless you use
-# self-signed certificates for your remote storage backend.
-
-# WEBDAV_URL_INSECURE="true"
-
 # In addition to storing backups remotely, you can also keep local copies.
 # Pass a container-local path to store your backups if needed. You also need to
 # mount a local folder or Docker volume into that location (`/archive`
@@ -364,6 +351,19 @@ You can populate below template according to your requirements and use it as you
 
 # LOCK_TIMEOUT="60m"
 
+########### HEADER FORMAT USED BY THE TAR ARCHIVE
+
+# By default, tar archive creation will pick a header format that is appropriate
+# for the archive's contents. In case you have special requirements or need to
+# work with tools that do not support all standard header formats, you can use
+# this option to enforce a certain header format. Valid options are "USTAR",
+# "PAX" and "GNU". Be aware that setting this value might create situations where
+# it's not possible to encode the information about a certain file, making the
+# backup fail.
+# In case no value is set, an appropriate format will be selected for each file.
+
+# TAR_ARCHIVE_HEADER_FORMAT="USTAR"
+
 ########### EMAIL NOTIFICATIONS
 
 # ************************************************************************
@@ -746,39 +746,6 @@ The exact order of schedules that use the same cron expression is not specified.
 In case you need your schedules to overlap, you need to create a dedicated container for each schedule instead.
 When changing the configuration, you currently need to manually restart the container for the changes to take effect.
 
-### Define different retention schedules
-
-If you want to manage backup retention on different schedules, the most straight forward approach is to define a dedicated configuration for retention rule using a different prefix in the `BACKUP_FILENAME` parameter and then run them on different cron schedules.
-
-For example, if you wanted to keep daily backups for 7 days, weekly backups for a month, and retain monthly backups forever, you could create three configuration files and mount them into `/etc/dockervolumebackup.d`:
-
-```ini
-# 01daily.conf
-BACKUP_FILENAME="daily-backup-%Y-%m-%dT%H-%M-%S.tar.gz"
-# run every day at 2am
-BACKUP_CRON_EXPRESSION="0 2 * * *"
-BACKUP_PRUNING_PREFIX="daily-backup-"
-BACKUP_RETENTION_DAYS="7"
-```
-
-```ini
-# 02weekly.conf
-BACKUP_FILENAME="weekly-backup-%Y-%m-%dT%H-%M-%S.tar.gz"
-# run every monday at 3am
-BACKUP_CRON_EXPRESSION="0 3 * * 1"
-BACKUP_PRUNING_PREFIX="weekly-backup-"
-BACKUP_RETENTION_DAYS="31"
-```
-
-```ini
-# 03monthly.conf
-BACKUP_FILENAME="monthly-backup-%Y-%m-%dT%H-%M-%S.tar.gz"
-# run every 1st of a month at 4am
-BACKUP_CRON_EXPRESSION="0 4 1 * *"
-```
-
-Note that while it's possible to define colliding cron schedules for each of these configurations, you might need to adjust the value for `LOCK_TIMEOUT` in case your backups are large and might take longer than an hour.
-
 ## Recipes
 
 This section lists configuration for some real-world use cases that you can mix and match according to your needs.

cmd/backup/archive.go

@@ -11,13 +11,14 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 )
 
-func createArchive(files []string, inputFilePath, outputFilePath string) error {
+func createArchive(inputFilePath, outputFilePath string, options createArchiveOptions) error {
 	inputFilePath = stripTrailingSlashes(inputFilePath)
 	inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
 	if err != nil {
@@ -27,7 +28,7 @@ func createArchive(files []string, inputFilePath, outputFilePath string) error {
 		return fmt.Errorf("createArchive: error creating output file path: %w", err)
 	}
 
-	if err := compress(files, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
+	if err := compress(inputFilePath, outputFilePath, filepath.Dir(inputFilePath), options); err != nil {
 		return fmt.Errorf("createArchive: error creating archive: %w", err)
 	}
 
@@ -51,7 +52,7 @@ func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error)
 	return inputFilePath, outputFilePath, err
 }
 
-func compress(paths []string, outFilePath, subPath string) error {
+func compress(inPath, outFilePath, subPath string, options createArchiveOptions) error {
 	file, err := os.Create(outFilePath)
 	if err != nil {
 		return fmt.Errorf("compress: error creating out file: %w", err)
@@ -61,8 +62,16 @@ func compress(paths []string, outFilePath, subPath string) error {
 	gzipWriter := gzip.NewWriter(file)
 	tarWriter := tar.NewWriter(gzipWriter)
 
+	var paths []string
+	if err := filepath.WalkDir(inPath, func(path string, di fs.DirEntry, err error) error {
+		paths = append(paths, path)
+		return err
+	}); err != nil {
+		return fmt.Errorf("compress: error walking filesystem tree: %w", err)
+	}
+
 	for _, p := range paths {
-		if err := writeTarGz(p, tarWriter, prefix); err != nil {
+		if err := writeTarGz(p, tarWriter, prefix, options.format); err != nil {
 			return fmt.Errorf("compress error writing %s to archive: %w", p, err)
 		}
 	}
@@ -85,7 +94,7 @@ func compress(paths []string, outFilePath, subPath string) error {
 	return nil
 }
 
-func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
+func writeTarGz(path string, tarWriter *tar.Writer, prefix string, format tar.Format) error {
 	fileInfo, err := os.Lstat(path)
 	if err != nil {
 		return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err)
@@ -104,6 +113,10 @@ func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
 	}
 
 	header, err := tar.FileInfoHeader(fileInfo, link)
+	if format >= 0 {
+		header.Format = format
+	}
+
 	if err != nil {
 		return fmt.Errorf("writeTarGz: error getting file info header: %w", err)
 	}
@@ -131,3 +144,7 @@ func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
 
 	return nil
 }
+
+type createArchiveOptions struct {
+	format tar.Format
+}

cmd/backup/config.go

@@ -4,8 +4,8 @@
 package main
 
 import (
+	"archive/tar"
 	"fmt"
-	"regexp"
 	"time"
 )
 
@@ -22,7 +22,6 @@ type Config struct {
 	BackupPruningPrefix        string        `split_words:"true"`
 	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
 	BackupFromSnapshot         bool          `split_words:"true"`
-	BackupExcludeRegexp        RegexpDecoder `split_words:"true"`
 	AwsS3BucketName            string        `split_words:"true"`
 	AwsS3Path                  string        `split_words:"true"`
 	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
@@ -41,27 +40,36 @@ type Config struct {
 	EmailSMTPUsername          string        `envconfig:"EMAIL_SMTP_USERNAME"`
 	EmailSMTPPassword          string        `envconfig:"EMAIL_SMTP_PASSWORD"`
 	WebdavUrl                  string        `split_words:"true"`
-	WebdavUrlInsecure          bool          `split_words:"true"`
 	WebdavPath                 string        `split_words:"true" default:"/"`
 	WebdavUsername             string        `split_words:"true"`
 	WebdavPassword             string        `split_words:"true"`
 	ExecLabel                  string        `split_words:"true"`
 	ExecForwardOutput          bool          `split_words:"true"`
 	LockTimeout                time.Duration `split_words:"true" default:"60m"`
+	TarArchiveHeaderFormat     TarFormat     `split_words:"true"`
 }
 
-type RegexpDecoder struct {
-	Re *regexp.Regexp
-}
+type TarFormat tar.Format
 
-func (r *RegexpDecoder) Decode(v string) error {
-	if v == "" {
+func (t *TarFormat) Decode(value string) error {
+	switch value {
+	case "PAX":
+		*t = TarFormat(tar.FormatPAX)
 		return nil
+	case "USTAR":
+		*t = TarFormat(tar.FormatUSTAR)
+		return nil
+	case "GNU":
+		*t = TarFormat(tar.FormatGNU)
+		return nil
+	case "":
+		*t = TarFormat(-1)
+		return nil
+	default:
+		return fmt.Errorf("tarFormat: unknown format %s", value)
 	}
-	re, err := regexp.Compile(v)
-	if err != nil {
-		return fmt.Errorf("config: error compiling given regexp `%s`: %w", v, err)
-	}
-	*r = RegexpDecoder{Re: re}
-	return nil
+}
+
+func (t *TarFormat) Format() tar.Format {
+	return tar.Format(*t)
 }

cmd/backup/script.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -147,15 +146,6 @@ func newScript() (*script, error) {
 		} else {
 			webdavClient := gowebdav.NewClient(s.c.WebdavUrl, s.c.WebdavUsername, s.c.WebdavPassword)
 			s.webdavClient = webdavClient
-			if s.c.WebdavUrlInsecure {
-				defaultTransport, ok := http.DefaultTransport.(*http.Transport)
-				if !ok {
-					return nil, errors.New("newScript: unexpected error when asserting type for http.DefaultTransport")
-				}
-				webdavTransport := defaultTransport.Clone()
-				webdavTransport.TLSClientConfig.InsecureSkipVerify = s.c.WebdavUrlInsecure
-				s.webdavClient.SetTransport(webdavTransport)
-			}
 		}
 	}
 
@@ -398,28 +388,9 @@ func (s *script) takeBackup() error {
 		s.logger.Infof("Removed tar file `%s`.", tarFile)
 		return nil
 	})
-
-	backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources))
-	if err != nil {
-		return fmt.Errorf("takeBackup: error getting absolute path: %w", err)
-	}
-
-	var filesEligibleForBackup []string
-	if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) {
-			return nil
-		}
-		filesEligibleForBackup = append(filesEligibleForBackup, path)
-		return nil
+	if err := createArchive(backupSources, tarFile, createArchiveOptions{
+		format: s.c.TarArchiveHeaderFormat.Format(),
 	}); err != nil {
-		return fmt.Errorf("compress: error walking filesystem tree: %w", err)
-	}
-
-	if err := createArchive(filesEligibleForBackup, backupSources, tarFile); err != nil {
 		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
 	}
 

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.16
 	github.com/otiai10/copy v1.7.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
+	github.com/studio-b12/gowebdav v0.0.0-20211109083228-3f8721cd4b6f
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 )

go.sum

@@ -660,8 +660,6 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/studio-b12/gowebdav v0.0.0-20211109083228-3f8721cd4b6f h1:L2NE7BXnSlSLoNYZ0lCwZDjdnYjCNYC71k9ClZUTFTs=
 github.com/studio-b12/gowebdav v0.0.0-20211109083228-3f8721cd4b6f/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
-github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9HY7giGM+kYcnQ71m14JnGdQabMPmyt++8=
-github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=

test/compose/docker-compose.yml

@@ -43,7 +43,6 @@ services:
       BACKUP_PRUNING_PREFIX: test
       GPG_PASSPHRASE: 1234secret
       WEBDAV_URL: http://webdav/
-      WEBDAV_URL_INSECURE: 'true'
       WEBDAV_PATH: /my/new/path/
       WEBDAV_USERNAME: test
       WEBDAV_PASSWORD: test

test/ignore/.gitignore

@@ -1 +0,0 @@
-local

test/ignore/docker-compose.yml

@@ -1,15 +0,0 @@
-version: '3.8'
-
-services:
-  backup:
-    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
-    deploy:
-      restart_policy:
-        condition: on-failure
-    environment:
-      BACKUP_FILENAME: test.tar.gz
-      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_EXCLUDE_REGEXP: '\.(me|you)$$'
-    volumes:
-      - ./local:/archive
-      - ./sources:/backup/data:ro

test/ignore/run.sh

@@ -1,27 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd $(dirname $0)
-mkdir -p local
-
-docker-compose up -d
-sleep 5
-docker-compose exec backup backup
-
-docker-compose down --volumes
-
-out=$(mktemp -d)
-sudo tar --same-owner -xvf ./local/test.tar.gz -C "$out"
-
-if [ ! -f "$out/backup/data/me.txt" ]; then
-  echo "[TEST:FAIL] Expected file was not found."
-  exit 1
-fi
-echo "[TEST:PASS] Expected file was found."
-
-if [ -f "$out/backup/data/skip.me" ]; then
-  echo "[TEST:FAIL] Ignored file was found."
-  exit 1
-fi
-echo "[TEST:PASS] Ignored file was not found."