Fix handling of empty directories (#44)

* Add test checking whether empty directories are included in backups

* Update targz library to include fix

README.md

@@ -122,6 +122,16 @@ You can populate below template according to your requirements and use it as you
 
 # BACKUP_FILENAME="backup-%Y-%m-%dT%H-%M-%S.tar.gz"
 
+# Setting BACKUP_FILENAME_EXPAND to true allows for environment variable
+# placeholders in BACKUP_FILENAME, BACKUP_LATEST_SYMLINK and in
+# BACKUP_PRUNING_PREFIX that will get expanded at runtime,
+# e.g. `backup-$HOSTNAME-%Y-%m-%dT%H-%M-%S.tar.gz`. Expansion happens before
+# interpolating strftime tokens. It is disabled by default.
+# Please note that you will need to escape the `$` when providing the value
+# in a docker-compose.yml file, i.e. using $$VAR instead of $VAR.
+
+# BACKUP_FILENAME_TEMPLATE="true"
+
 # When storing local backups, a symlink to the latest backup can be created
 # in case a value is given for this key. This has no effect on remote backups.
 
@@ -286,6 +296,11 @@ You can populate below template according to your requirements and use it as you
 # EMAIL_SMTP_PORT="<port>"
 ```
 
+In case you encouter double quoted values in your configuration you might be running an [older version of `docker-compose`].
+You can work around this by either updating `docker-compose` or unquoting your configuration values.
+
+[compose-issue]: https://github.com/docker/compose/issues/2854
+
 ## How to
 
 ### Stopping containers during backup

cmd/backup/main.go

@@ -103,6 +103,7 @@ type script struct {
 type config struct {
 	BackupSources              string        `split_words:"true" default:"/backup"`
 	BackupFilename             string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
+	BackupFilenameExpand       bool          `split_words:"true"`
 	BackupLatestSymlink        string        `split_words:"true"`
 	BackupArchive              string        `split_words:"true" default:"/archive"`
 	BackupRetentionDays        int32         `split_words:"true" default:"-1"`
@@ -153,6 +154,12 @@ func newScript() (*script, error) {
 	}
 
 	s.file = path.Join("/tmp", s.c.BackupFilename)
+	if s.c.BackupFilenameExpand {
+		s.file = os.ExpandEnv(s.file)
+		s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
+		s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix)
+	}
+	s.file = timeutil.Strftime(&s.start, s.file)
 
 	_, err := os.Stat("/var/run/docker.sock")
 	if !os.IsNotExist(err) {
@@ -413,7 +420,6 @@ func (s *script) stopContainers() (func() error, error) {
 // takeBackup creates a tar archive of the configured backup location and
 // saves it to disk.
 func (s *script) takeBackup() error {
-	s.file = timeutil.Strftime(&s.start, s.file)
 	backupSources := s.c.BackupSources
 
 	if s.c.BackupFromSnapshot {

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/gofrs/flock v0.8.1
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
-	github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615
+	github.com/m90/targz v0.0.0-20211229090208-2f22c2d9278e
 	github.com/minio/minio-go/v7 v7.0.16
 	github.com/otiai10/copy v1.7.0
 	github.com/sirupsen/logrus v1.8.1

go.sum

@@ -450,8 +450,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs=
 github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615 h1:rn0LO2tQEgCDOct8qnbcslTUpAIWdVlWcGkjoumhf2U=
-github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615/go.mod h1:YZK3bSO/oVlk9G+v00BxgzxW2Us4p/R4ysHOBjk0fJI=
+github.com/m90/targz v0.0.0-20211229090208-2f22c2d9278e h1:Kzm2zfxS40RUGD5UVtVtOo9RT5TtGoNJnmWORtCEaxM=
+github.com/m90/targz v0.0.0-20211229090208-2f22c2d9278e/go.mod h1:YZK3bSO/oVlk9G+v00BxgzxW2Us4p/R4ysHOBjk0fJI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=

test/cli/run.sh

@@ -7,6 +7,7 @@ cd $(dirname $0)
 docker network create test_network
 docker volume create backup_data
 docker volume create app_data
+docker volume create empty_data
 
 docker run -d \
   --name minio \
@@ -31,6 +32,7 @@ sleep 10
 docker run --rm \
   --network test_network \
   -v app_data:/backup/app_data \
+  -v empty_data:/backup/empty_data \
   -v /var/run/docker.sock:/var/run/docker.sock \
   --env AWS_ACCESS_KEY_ID=test \
   --env AWS_SECRET_ACCESS_KEY=GMusLtUmILge2by+z890kQ \
@@ -44,7 +46,7 @@ docker run --rm \
 
 docker run --rm -it \
   -v backup_data:/data alpine \
-  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db'
+  ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db && test -d /backup/empty_data'
 
 echo "[TEST:PASS] Found relevant files in untared backup."
 

test/compose/docker-compose.yml

@@ -14,6 +14,7 @@ services:
 
   backup: &default_backup_service
     image: offen/docker-volume-backup:${TEST_VERSION}
+    hostname: hostnametoken
     depends_on:
       - minio
     restart: always
@@ -23,8 +24,9 @@ services:
       AWS_ENDPOINT: minio:9000
       AWS_ENDPOINT_PROTO: http
       AWS_S3_BUCKET_NAME: backup
-      BACKUP_FILENAME: test.tar.gz
-      BACKUP_LATEST_SYMLINK: test.latest.tar.gz.gpg
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
       BACKUP_PRUNING_LEEWAY: 5s

test/compose/run.sh

@@ -14,12 +14,12 @@ docker-compose exec backup backup
 
 docker run --rm -it \
   -v compose_backup_data:/data alpine \
-  ash -c 'apk add gnupg && echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /data/backup/test.tar.gz.gpg > /tmp/test.tar.gz && tar -xf /tmp/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+  ash -c 'apk add gnupg && echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /data/backup/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
 
 echo "[TEST:PASS] Found relevant files in untared remote backup."
 
-test -L ./local/test.latest.tar.gz.gpg
-echo 1234secret | gpg -d --yes --passphrase-fd 0 ./local/test.tar.gz.gpg > ./local/decrypted.tar.gz
+test -L ./local/test-hostnametoken.latest.tar.gz.gpg
+echo 1234secret | gpg -d --yes --passphrase-fd 0 ./local/test-hostnametoken.tar.gz.gpg > ./local/decrypted.tar.gz
 tar -xf ./local/decrypted.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
 rm ./local/decrypted.tar.gz
 test -L /tmp/backup/app_data/db.link