Update existing tests and test actual pruning

cmd/backup/config.go

@@ -38,6 +38,7 @@ type Config struct {
 	BackupArchive                 string          `split_words:"true" default:"/archive"`
 	BackupCronExpression          string          `split_words:"true" default:"@daily"`
 	BackupRetentionDays           int32           `split_words:"true" default:"-1"`
+	BackupRetentionPeriod         time.Duration   `split_words:"true"`
 	BackupPruningLeeway           time.Duration   `split_words:"true" default:"1m"`
 	BackupPruningPrefix           string          `split_words:"true"`
 	BackupStopContainerLabel      string          `split_words:"true"`

cmd/backup/prune_backups.go

@@ -17,11 +17,18 @@ import (
 // the given configuration. In case the given configuration would delete all
 // backups, it does nothing instead and logs a warning.
 func (s *script) pruneBackups() error {
-	if s.c.BackupRetentionDays < 0 {
+	if s.c.BackupRetentionDays < 0 && s.c.BackupRetentionPeriod == 0 {
 		return nil
 	}
 
-	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
+	var deadline time.Time
+	if s.c.BackupRetentionPeriod != 0 {
+		deadline = time.Now().Add(-s.c.BackupRetentionPeriod)
+	} else {
+		s.logger.Warn("Using BACKUP_RETENTION_DAYS has been deprecated and will be removed in the next major version. Please use BACKUP_RETENTION_PERIOD instead.")
+		deadline = time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays))
+	}
+	deadline = deadline.Add(s.c.BackupPruningLeeway)
 
 	eg := errgroup.Group{}
 	for _, backend := range s.storages {

cmd/backup/script.go

@@ -225,6 +225,10 @@ func (s *script) init() error {
 		s.storages = append(s.storages, dropboxBackend)
 	}
 
+	if s.c.BackupRetentionDays > -1 && s.c.BackupRetentionPeriod > 0 {
+		return errwrap.Wrap(nil, "both BACKUP_RETENTION_DAYS and BACKUP_RETENTION_PERIOD were configured, which are mutually exclusive")
+	}
+
 	if s.c.EmailNotificationRecipient != "" {
 		emailURL := fmt.Sprintf(
 			"smtp://%s:%s@%s:%d/?from=%s&to=%s",

docs/how-tos/automatically-prune-old-backups.md

@@ -7,7 +7,8 @@ nav_order: 3
 
 # Automatically prune old backups
 
-When `BACKUP_RETENTION_DAYS` is configured, the command will check if there are any archives in the remote storage backend(s) or local archive that are older than the given retention value and rotate these backups away.
+When `BACKUP_RETENTION_PERIOD` is configured, the command will check if there are any archives in the remote storage backend(s) or local archive that are older than the given retention value and rotate these backups away.
+The value is a duration as per Go's [`time.ParseDuration`][duration].
 
 {: .note }
 Be aware that this mechanism looks at __all files in the target bucket or archive__, which means that other files that are older than the given deadline are deleted as well.
@@ -23,7 +24,7 @@ services:
     environment:
       BACKUP_FILENAME: backup-%Y-%m-%dT%H-%M-%S.tar.gz
       BACKUP_PRUNING_PREFIX: backup-
-      BACKUP_RETENTION_DAYS: '7'
+      BACKUP_RETENTION_PERIOD: '168h'
     volumes:
       - ${HOME}/backups:/archive
       - data:/backup/my-app-backup:ro
@@ -32,3 +33,5 @@ services:
 volumes:
   data:
 ```
+
+[duration]: https://pkg.go.dev/time#ParseDuration

docs/recipes/index.md

@@ -280,7 +280,7 @@ services:
       AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       BACKUP_FILENAME: backup-%Y-%m-%dT%H-%M-%S.tar.gz
       BACKUP_PRUNING_PREFIX: backup-
-      BACKUP_RETENTION_DAYS: 7
+      BACKUP_RETENTION_PERIOD: 168h
     volumes:
       - data:/backup/my-app-backup:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro

docs/reference/index.md

@@ -312,9 +312,10 @@ You can populate below template according to your requirements and use it as you
 # removal to certain files.
 
 # Define this value to enable automatic rotation of old backups. The value
-# declares the number of days for which a backup is kept.
+# declares the duration for which a backup is kept. It is formatted as per
+# https://pkg.go.dev/time#ParseDuration, e.g. 1 day turns into `24h`
 
-# BACKUP_RETENTION_DAYS="7"
+# BACKUP_RETENTION_PERIOD="168h"
 
 # In case the duration a backup takes fluctuates noticeably in your setup
 # you can adjust this setting to make sure there are no race conditions

test/age-passphrase/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_FILENAME: test.tar.gz
       BACKUP_LATEST_SYMLINK: test-latest.tar.gz.age
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       AGE_PASSPHRASE: "Dance.0Tonight.Go.Typical"
     volumes:
       - ${LOCAL_DIR:-./local}:/archive

test/age-publickey/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_FILENAME: test.tar.gz
       BACKUP_LATEST_SYMLINK: test-latest.tar.gz.age
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       AGE_PUBLIC_KEYS: "${BACKUP_AGE_PUBLIC_KEYS}"
     volumes:
       - ${LOCAL_DIR:-./local}:/archive

test/azure/docker-compose.yml

@@ -37,7 +37,7 @@ services:
       AZURE_STORAGE_ACCESS_TIER: Hot
       BACKUP_FILENAME: test.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
     volumes:

test/azure/run.sh

@@ -38,7 +38,7 @@ rm "$LOCAL_DIR/test.tar.gz"
 
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
-BACKUP_RETENTION_DAYS="0" docker compose up -d
+BACKUP_RETENTION_PERIOD="1s" docker compose up -d
 sleep 5
 
 docker compose exec backup backup
@@ -52,7 +52,7 @@ pass "Remote backups have not been deleted."
 # The third part of this test checks if old backups get deleted when the retention
 # is set to 7 days (which it should)
 
-BACKUP_RETENTION_DAYS="7" docker compose up -d
+BACKUP_RETENTION_PERIOD="168h" docker compose up -d
 sleep 5
 
 info "Create first backup with no prune"

test/certs/docker-compose.yml

@@ -26,7 +26,7 @@ services:
       AWS_ENDPOINT_CA_CERT: /root/minio-rootCA.crt
       AWS_S3_BUCKET_NAME: backup
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       BACKUP_PRUNING_LEEWAY: 5s
     volumes:
       - app_data:/backup/app_data:ro

test/dropbox/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       BACKUP_FILENAME_EXPAND: 'true'
       BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
       DROPBOX_ENDPOINT: http://openapi_mock:8080

test/dropbox/run.sh

@@ -29,7 +29,7 @@ fi
 
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
-BACKUP_RETENTION_DAYS="0" docker compose up -d
+BACKUP_RETENTION_PERIOD="1s" docker compose up -d
 sleep 5
 
 logs=$(docker compose exec -T backup backup)
@@ -43,7 +43,7 @@ fi
 
 # The third part of this test checks if old backups get deleted when the retention
 # is set to 7 days (which it should)
-BACKUP_RETENTION_DAYS="7" docker compose up -d
+BACKUP_RETENTION_PERIOD="168h" docker compose up -d
 sleep 5
 
 info "Create second backup and prune"

test/gpg-asym/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_FILENAME: test.tar.gz
       BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       GPG_PUBLIC_KEY_RING_FILE: /keys/public_key.asc
     volumes:
       - ${KEY_DIR:-.}/public_key.asc:/keys/public_key.asc

test/gpg/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_FILENAME: test.tar.gz
       BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       GPG_PASSPHRASE: 1234#$$ecret
     volumes:
       - ${LOCAL_DIR:-./local}:/archive

test/local/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
       BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
     volumes:

test/local/run.sh

@@ -41,7 +41,7 @@ pass "Found symlink to latest version in local backup."
 
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
-BACKUP_RETENTION_DAYS="0" docker compose up -d
+BACKUP_RETENTION_PERIOD="1s" docker compose up -d
 sleep 5
 
 docker compose exec backup backup
@@ -54,7 +54,7 @@ pass "Local backups have not been deleted."
 # The third part of this test checks if old backups get deleted when the retention
 # is set to 7 days (which it should)
 
-BACKUP_RETENTION_DAYS="7" docker compose up -d
+BACKUP_RETENTION_PERIOD="168h" docker compose up -d
 sleep 5
 
 info "Create first backup with no prune"

test/lock/docker-compose.yml

@@ -4,7 +4,7 @@ services:
     restart: always
     environment:
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: '7'
+      BACKUP_RETENTION_PERIOD: 168h
     volumes:
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock

test/lock/run.sh

@@ -13,7 +13,7 @@ sleep 5
 
 ec=0
 
-docker compose exec -e BACKUP_RETENTION_DAYS=7 -e BACKUP_FILENAME=test.tar.gz backup backup & \
+docker compose exec -e BACKUP_RETENTION_PERIOD=168h -e BACKUP_FILENAME=test.tar.gz backup backup & \
   { set +e; sleep 0.1; docker compose exec -e BACKUP_FILENAME=test2.tar.gz -e LOCK_TIMEOUT=1s backup backup; ec=$?;}
 
 if [ "$ec" = "0" ]; then

test/pruning/docker-compose.yml

@@ -25,7 +25,7 @@ services:
       BACKUP_FILENAME_EXPAND: 'true'
       BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: 7
+      BACKUP_RETENTION_PERIOD: 168h
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
       BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz

test/retention/docker-compose.yml

@@ -0,0 +1,22 @@
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    environment:
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_PERIOD: 15s
+      BACKUP_PRUNING_LEEWAY: 1s
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${LOCAL_DIR:-./local}:/archive
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/retention/run.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+
+docker compose exec backup backup
+
+sleep 20
+
+if [ $(ls -1 $LOCAL_DIR | wc -l) != "1" ]; then
+  fail "Unexpected number of backups after initial run"
+fi
+pass "Found 1 backup files."
+
+docker compose exec backup backup
+
+if [ $(ls -1 $LOCAL_DIR | wc -l) != "1" ]; then
+  fail "Unexpected number of backups after initial run"
+fi
+pass "Found 1 backup files."

test/s3/docker-compose.yml

@@ -25,7 +25,7 @@ services:
       BACKUP_FILENAME_EXPAND: 'true'
       BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
     volumes:

test/s3/run.sh

@@ -22,9 +22,11 @@ docker run --rm \
 
 pass "Found relevant files in untared remote backups."
 
+sleep 5
+
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
-BACKUP_RETENTION_DAYS="0" docker compose up -d
+BACKUP_RETENTION_PERIOD="5s" docker compose up -d
 sleep 5
 
 docker compose exec backup backup
@@ -39,7 +41,7 @@ pass "Remote backups have not been deleted."
 # The third part of this test checks if old backups get deleted when the retention
 # is set to 7 days (which it should)
 
-BACKUP_RETENTION_DAYS="7" docker compose up -d
+BACKUP_RETENTION_PERIOD="168h" docker compose up -d
 sleep 5
 
 info "Create first backup with no prune"

test/secrets/docker-compose.yml

@@ -31,7 +31,7 @@ services:
       AWS_S3_BUCKET_NAME: backup
       BACKUP_FILENAME: test.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: 7
+      BACKUP_RETENTION_PERIOD: 168h
       BACKUP_PRUNING_LEEWAY: 5s
     volumes:
       - pg_data:/backup/pg_data:ro

test/services/docker-compose.yml

@@ -25,7 +25,7 @@ services:
       AWS_S3_BUCKET_NAME: backup
       BACKUP_FILENAME: test.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: 7
+      BACKUP_RETENTION_PERIOD: 168h
       BACKUP_PRUNING_LEEWAY: 5s
     volumes:
       - pg_data:/backup/pg_data:ro

test/ssh/docker-compose.yml

@@ -19,7 +19,7 @@ services:
       BACKUP_FILENAME_EXPAND: 'true'
       BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
       SSH_HOST_NAME: ssh

test/ssh/run.sh

@@ -28,7 +28,7 @@ pass "Found relevant files in decrypted and untared remote backups."
 
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
-BACKUP_RETENTION_DAYS="0" docker compose up -d
+BACKUP_RETENTION_PERIOD="1s" docker compose up -d
 sleep 5
 
 docker compose exec backup backup
@@ -43,7 +43,7 @@ pass "Remote backups have not been deleted."
 # The third part of this test checks if old backups get deleted when the retention
 # is set to 7 days (which it should)
 
-BACKUP_RETENTION_DAYS="7" docker compose up -d
+BACKUP_RETENTION_PERIOD="168h" docker compose up -d
 sleep 5
 
 info "Create first backup with no prune"

test/swarm/docker-compose.yml

@@ -31,7 +31,7 @@ services:
       AWS_S3_BUCKET_NAME: backup
       BACKUP_FILENAME: test.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: 7
+      BACKUP_RETENTION_PERIOD: 168h
       BACKUP_PRUNING_LEEWAY: 5s
     volumes:
       - pg_data:/backup/pg_data:ro

test/webdav/docker-compose.yml

@@ -18,7 +18,7 @@ services:
       BACKUP_FILENAME_EXPAND: 'true'
       BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
-      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h}
       BACKUP_PRUNING_LEEWAY: 5s
       BACKUP_PRUNING_PREFIX: test
       WEBDAV_URL: http://webdav/

test/webdav/run.sh

@@ -24,7 +24,7 @@ pass "Found relevant files in untared remote backup."
 
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
-BACKUP_RETENTION_DAYS="0" docker compose up -d
+BACKUP_RETENTION_PERIOD="1s" docker compose up -d
 sleep 5
 
 docker compose exec backup backup
@@ -39,7 +39,7 @@ pass "Remote backups have not been deleted."
 # The third part of this test checks if old backups get deleted when the retention
 # is set to 7 days (which it should)
 
-BACKUP_RETENTION_DAYS="7" docker compose up -d
+BACKUP_RETENTION_PERIOD="168h" docker compose up -d
 sleep 5
 
 info "Create first backup with no prune"