Consider prefix rules when pruning WebDAV storages (#79)

* Allow mounting of config directory for multiple schedules

* Add docs for conf.d feature

* Fix behavior on multiple files

* Define default case first in entrypoint script

Dockerfile

@@ -14,7 +14,7 @@ FROM alpine:3.15
 
 WORKDIR /root
 
-RUN apk add --update ca-certificates
+RUN apk add --no-cache ca-certificates
 
 COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
 

README.md

@@ -28,6 +28,7 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
   - [Manually triggering a backup](#manually-triggering-a-backup)
   - [Update deprecated email configuration](#update-deprecated-email-configuration)
   - [Using a custom Docker host](#using-a-custom-docker-host)
+  - [Run multiple backup schedules in the same container](#run-multiple-backup-schedules-in-the-same-container)
 - [Recipes](#recipes)
   - [Backing up to AWS S3](#backing-up-to-aws-s3)
   - [Backing up to Filebase](#backing-up-to-filebase)
@@ -155,6 +156,11 @@ You can populate below template according to your requirements and use it as you
 
 # BACKUP_FROM_SNAPSHOT="false"
 
+# By default, the `/backup` directory inside the container will be backed up.
+# In case you need to use a custom location, set `BACKUP_SOURCES`.
+
+# BACKUP_SOURCES="/other/location"
+
 ########### BACKUP STORAGE
 
 # The name of the remote bucket that should be used for storing backups. If
@@ -555,6 +561,26 @@ In case you need to restore a volume from a backup, the most straight forward pr
 
 Depending on your setup and the application(s) you are running, this might involve other steps to be taken still.
 
+---
+
+If you want to rollback an entire volume to an earlier backup snapshot (recommended for database volumes):
+
+- Trigger a manual backup if necessary (see `Manually triggering a backup`).
+- Stop the container(s) that are using the volume.
+- If volume was initially created using docker-compose, find out exact volume name using:
+  ```console
+  docker volume ls
+  ```
+- Remove existing volume (the example assumes it's named `data`):
+  ```console
+  docker volume rm data
+  ```
+- Create new volume with the same name and restore a snapshot:
+  ```console
+  docker run --rm -it -v data:/backup/my-app-backup -v /path/to/local_backups:/archive:ro alpine tar -xvzf /archive/full_backup_filename.tar.gz
+  ```
+- Restart the container(s) that are using the volume.
+
 ### Set the timezone the container runs in
 
 By default a container based on this image will run in the UTC timezone.
@@ -631,7 +657,33 @@ If you are interfacing with Docker via TCP, set `DOCKER_HOST` to the correct URL
 DOCKER_HOST=tcp://docker_socket_proxy:2375
 ```
 
-In case you are using a socket proxy, it must support `GET` and `POST` requests to the `/containers` endpoint. If you are using Docker Swarm, it must also support the `/services` endpoint.
+In case you are using a socket proxy, it must support `GET` and `POST` requests to the `/containers` endpoint. If you are using Docker Swarm, it must also support the `/services` endpoint. If you are using pre/post backup commands, it must also support the `/exec` endpoint.
+
+### Run multiple backup schedules in the same container
+
+Multiple backup schedules with different configuration can be configured by mounting an arbitrary number of configuration files (using the `.env` format) into `/etc/dockervolumebackup/conf.d`:
+
+```yml
+version: '3'
+
+services:
+  # ... define other services using the `data` volume here
+  backup:
+    image: offen/docker-volume-backup:latest
+    volumes:
+      - data:/backup/my-app-backup:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./configuration:/etc/dockervolumebackup/conf.d
+
+volumes:
+  data:
+```
+
+A separate cronjob will be created for each config file.
+If a configuration value is set both in the global environment as well as in the config file, the config file will take precedence.
+The `backup` command expects to run on an exclusive lock, so it is your responsibility to make sure the invocations do not overlap.
+In case you need your schedules to overlap, you need to create a dedicated container for each schedule instead.
+When changing the configuration, you currently need to manually restart the container for the changes to take effect.
 
 ## Recipes
 

cmd/backup/exec.go

@@ -1,6 +1,9 @@
 // Copyright 2022 - Offen Authors <hioffen@posteo.de>
 // SPDX-License-Identifier: MPL-2.0
 
+// Portions of this file are taken and adapted from `moby`, Copyright 2012-2017 Docker, Inc.
+// Licensed under the Apache 2.0 License: https://github.com/moby/moby/blob/8e610b2b55bfd1bfa9436ab110d311f5e8a74dcb/LICENSE
+
 package main
 
 import (
@@ -87,7 +90,7 @@ func (s *script) runLabeledCommands(label string) error {
 		Filters: filters.NewArgs(f...),
 	})
 	if err != nil {
-		return fmt.Errorf("runLabeledCommands: error querying for containers", err)
+		return fmt.Errorf("runLabeledCommands: error querying for containers: %w", err)
 	}
 
 	if len(containersWithCommand) == 0 {

cmd/backup/script.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"text/template"
 	"time"
 
@@ -580,6 +581,9 @@ func (s *script) pruneBackups() error {
 		var matches []fs.FileInfo
 		var lenCandidates int
 		for _, candidate := range candidates {
+			if !strings.HasPrefix(candidate.Name(), s.c.BackupPruningPrefix) {
+				continue
+			}
 			lenCandidates++
 			if candidate.ModTime().Before(deadline) {
 				matches = append(matches, candidate)

entrypoint.sh

@@ -5,10 +5,21 @@
 
 set -e
 
-BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+if [ ! -d "/etc/dockervolumebackup/conf.d" ]; then
+  BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
 
-echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
-echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
+  echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
+  echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
+else
+  echo "/etc/dockervolumebackup/conf.d was found, using configuration files from this directory."
+
+  for file in /etc/dockervolumebackup/conf.d/*; do
+    source $file
+    BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
+    echo "Appending cron.d entry with expression $BACKUP_CRON_EXPRESSION and configuration file $file"
+    (crontab -l; echo "$BACKUP_CRON_EXPRESSION /bin/sh -c 'set -a; source $file; set +a && backup' 2>&1") | crontab -
+  done
+fi
 
 echo "Starting cron in foreground."
 crond -f -l 8

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/containrrr/shoutrrr v0.5.2
+	github.com/cosiner/argv v0.1.0
 	github.com/docker/docker v20.10.11+incompatible
 	github.com/gofrs/flock v0.8.1
 	github.com/kelseyhightower/envconfig v1.4.0
@@ -18,7 +19,6 @@ require (
 require (
 	github.com/Microsoft/go-winio v0.4.17 // indirect
 	github.com/containerd/containerd v1.5.5 // indirect
-	github.com/cosiner/argv v0.1.0 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect

test/compose/docker-compose.yml

@@ -21,7 +21,7 @@ services:
     volumes:
       - webdav_backup_data:/var/lib/dav
 
-  backup: &default_backup_service
+  backup:
     image: offen/docker-volume-backup:${TEST_VERSION:-canary}
     hostname: hostnametoken
     depends_on:

test/confd/.gitignore

@@ -0,0 +1 @@
+local

test/confd/backup.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="conf.tar.gz"
+BACKUP_CRON_EXPRESSION="*/1 * * * *"

test/confd/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    volumes:
+      - ./local:/archive
+      - app_data:/backup/app_data:ro
+      - ./backup.env:/etc/dockervolumebackup/conf.d/00backup.env
+      - ./never.env:/etc/dockervolumebackup/conf.d/10never.env
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/confd/never.env

@@ -0,0 +1,2 @@
+BACKUP_FILENAME="never.tar.gz"
+BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?"

test/confd/run.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+
+mkdir -p local
+
+docker-compose up -d
+
+# sleep until a backup is guaranteed to have happened on the 1 minute schedule
+sleep 100
+
+docker-compose down --volumes
+
+if [ ! -f ./local/conf.tar.gz ]; then
+  echo "[TEST:FAIL] Config from file was not used."
+  exit 1
+fi
+echo "[TEST:PASS] Config from file was used."
+
+if [ -f ./local/never.tar.gz ]; then
+  echo "[TEST:FAIL] Unexpected file was found."
+  exit 1
+fi
+echo "[TEST:PASS] Unexpected cron did not run."