Improve Swarm support (#333)

* Query for labeled services as well

* Try scaling down services

* Scale services back up

* Use progress tool from Docker CLI

* In test, label both services

* Clean up error and log messages

* Document scale-up/down approach in docs

* Downgrade Docker CLI to match client

* Document services stats

* Do not rely on PreviousSpec for storing desired replica count

* Log warnings from Docker when updating services

* Check whether container and service labels collide

* Document script behavior on label collision

* Add additional check if all containers have been removed

* Scale services concurrently

* Move docker interaction code into own file

* Factor out code for service updating

* Time out after five minutes of not reaching desired container count

* Inline handling of in-swarm container level restart

* Timer is more suitable for timeout race

* Timeout when scaling down services should be configurable

* Choose better filename

* Reflect changes in naming

* Rename and deprecate BACKUP_STOP_CONTAINER_LABEL

* Improve logging

* Further simplify logging

.github/workflows/deploy-docs.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:

.github/workflows/golangci-lint.yml

@@ -15,8 +15,8 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: '1.21'
           cache: false

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
       contents: read
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
   test:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2

cmd/backup/config.go

@@ -37,7 +37,9 @@ type Config struct {
 	BackupRetentionDays           int32           `split_words:"true" default:"-1"`
 	BackupPruningLeeway           time.Duration   `split_words:"true" default:"1m"`
 	BackupPruningPrefix           string          `split_words:"true"`
-	BackupStopContainerLabel      string          `split_words:"true" default:"true"`
+	BackupStopContainerLabel      string          `split_words:"true"`
+	BackupStopDuringBackupLabel   string          `split_words:"true" default:"true"`
+	BackupStopServiceTimeout      time.Duration   `split_words:"true" default:"5m"`
 	BackupFromSnapshot            bool            `split_words:"true"`
 	BackupExcludeRegexp           RegexpDecoder   `split_words:"true"`
 	BackupSkipBackendsFromPrune   []string        `split_words:"true"`

cmd/backup/main.go

@@ -21,6 +21,9 @@ func main() {
 	defer func() {
 		if pArg := recover(); pArg != nil {
 			if err, ok := pArg.(error); ok {
+				s.logger.Error(
+					fmt.Sprintf("Executing the script encountered a panic: %v", err),
+				)
 				if hookErr := s.runHooks(err); hookErr != nil {
 					s.logger.Error(
 						fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr),
@@ -44,12 +47,12 @@ func main() {
 	}()
 
 	s.must(s.withLabeledCommands(lifecyclePhaseArchive, func() error {
-		restartContainers, err := s.stopContainersAndServices()
+		restartContainersAndServices, err := s.stopContainersAndServices()
 		// The mechanism for restarting containers is not using hooks as it
 		// should happen as soon as possible (i.e. before uploading backups or
 		// similar).
 		defer func() {
-			s.must(restartContainers())
+			s.must(restartContainersAndServices())
 		}()
 		if err != nil {
 			return err

cmd/backup/script.go

@@ -322,7 +322,7 @@ func (s *script) createArchive() error {
 			"Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.",
 		)
 		s.logger.Warn(
-			"Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the README for an upgrade guide.",
+			"Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the documentation for an upgrade guide.",
 		)
 		backupSources = filepath.Join("/tmp", s.c.BackupSources)
 		// copy before compressing guard against a situation where backup folder's content are still growing.

cmd/backup/stop_restart.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"sync"
 	"time"
 
@@ -41,9 +42,9 @@ func scaleService(cli *client.Client, serviceID string, replicas uint64) ([]stri
 	return response.Warnings, nil
 }
 
-func awaitContainerCountForService(cli *client.Client, serviceID string, count int) error {
+func awaitContainerCountForService(cli *client.Client, serviceID string, count int, timeoutAfter time.Duration) error {
 	poll := time.NewTicker(time.Second)
-	timeout := time.NewTicker(5 * time.Minute)
+	timeout := time.NewTimer(timeoutAfter)
 	defer timeout.Stop()
 	defer poll.Stop()
 
@@ -51,7 +52,8 @@ func awaitContainerCountForService(cli *client.Client, serviceID string, count i
 		select {
 		case <-timeout.C:
 			return fmt.Errorf(
-				"awaitContainerCount: timed out after waiting 5 minutes for service %s to reach desired container count of %d",
+				"awaitContainerCount: timed out after waiting %s for service %s to reach desired container count of %d",
+				timeoutAfter,
 				serviceID,
 				count,
 			)
@@ -86,9 +88,23 @@ func (s *script) stopContainersAndServices() (func() error, error) {
 	}
 	isDockerSwarm := dockerInfo.Swarm.LocalNodeState != "inactive"
 
+	labelValue := s.c.BackupStopDuringBackupLabel
+	if s.c.BackupStopContainerLabel != "" {
+		s.logger.Warn(
+			"Using BACKUP_STOP_CONTAINER_LABEL has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use BACKUP_STOP_DURING_BACKUP_LABEL instead. Refer to the docs for an upgrade guide.",
+		)
+		if _, ok := os.LookupEnv("BACKUP_STOP_DURING_BACKUP_LABEL"); ok {
+			return noop, errors.New("(*script).stopContainersAndServices: both BACKUP_STOP_DURING_BACKUP_LABEL and BACKUP_STOP_CONTAINER_LABEL have been set, cannot continue")
+		}
+		labelValue = s.c.BackupStopContainerLabel
+	}
+
 	filterMatchLabel := fmt.Sprintf(
 		"docker-volume-backup.stop-during-backup=%s",
-		s.c.BackupStopContainerLabel,
+		labelValue,
 	)
 
 	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{})
@@ -156,14 +172,22 @@ func (s *script) stopContainersAndServices() (func() error, error) {
 
 	s.logger.Info(
 		fmt.Sprintf(
-			"Stopping %d out of %d running container(s) and scaling down %d out of %d active service(s) as they were labeled %s.",
+			"Stopping %d out of %d running container(s) as they were labeled %s.",
 			len(containersToStop),
 			len(allContainers),
-			len(servicesToScaleDown),
-			len(allServices),
 			filterMatchLabel,
 		),
 	)
+	if isDockerSwarm {
+		s.logger.Info(
+			fmt.Sprintf(
+				"Scaling down %d out of %d active service(s) as they were labeled %s.",
+				len(servicesToScaleDown),
+				len(allServices),
+				filterMatchLabel,
+			),
+		)
+	}
 
 	var stoppedContainers []types.Container
 	var stopErrors []error
@@ -196,7 +220,7 @@ func (s *script) stopContainersAndServices() (func() error, error) {
 				}
 				// progress.ServiceProgress returns too early, so we need to manually check
 				// whether all containers belonging to the service have actually been removed
-				if err := awaitContainerCountForService(s.cli, svc.serviceID, 0); err != nil {
+				if err := awaitContainerCountForService(s.cli, svc.serviceID, 0, s.c.BackupStopServiceTimeout); err != nil {
 					scaleDownErrors.append(err)
 				}
 			}(svc)
@@ -293,13 +317,22 @@ func (s *script) stopContainersAndServices() (func() error, error) {
 				errors.Join(allErrors...),
 			)
 		}
+
 		s.logger.Info(
 			fmt.Sprintf(
-				"Restarted %d container(s) and %d service(s).",
+				"Restarted %d container(s).",
 				len(stoppedContainers),
-				len(scaledDownServices),
 			),
 		)
+		if isDockerSwarm {
+			s.logger.Info(
+				fmt.Sprintf(
+					"Scaled %d service(s) back up.",
+					len(scaledDownServices),
+				),
+			)
+		}
+
 		return nil
 	}, initialErr
 }

docs/how-tos/replace-deprecated-backup-stop-container-label.md

@@ -0,0 +1,19 @@
+---
+title: Replace deprecated BACKUP_STOP_CONTAINER_LABEL setting
+layout: default
+parent: How Tos
+nav_order: 19
+---
+
+# Replace deprecated `BACKUP_STOP_CONTAINER_LABEL` setting
+
+Version `v2.36.0` deprecated the `BACKUP_STOP_CONTAINER_LABEL` setting and renamed it `BACKUP_STOP_DURING_BACKUP_LABEL` which is supposed to signal that this will stop both containers _and_ services.
+Migrating is done by renaming the key for your custom value:
+
+```diff
+    env:
+-     BACKUP_STOP_CONTAINER_LABEL: database
++     BACKUP_STOP_DURING_BACKUP_LABEL: database
+```
+
+The old key will stay supported until the next major version, but logs a warning each time a backup is taken.

docs/how-tos/set-up-notifications.md

@@ -76,7 +76,7 @@ Configuration, data about the backup run and helper functions will be passed to
 
 Here is a list of all data passed to the template:
 
-* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`)
+* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_DURING_BACKUP_LABEL` becomes `BackupStopDuringBackupLabel`)
 * `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates
 * `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available.
   * `StartTime`: time when the script started execution

docs/how-tos/stop-containers-during-backup.md

@@ -14,7 +14,7 @@ In many cases, it will be desirable to stop the services that are consuming the
 This image can automatically stop and restart containers and services.
 By default, any container that is labeled `docker-volume-backup.stop-during-backup=true` will be stopped before the backup is being taken and restarted once it has finished.
 
-In case you need more fine grained control about which containers should be stopped (e.g. when backing up multiple volumes on different schedules), you can set the `BACKUP_STOP_CONTAINER_LABEL` environment variable and then use the same value for labeling:
+In case you need more fine grained control about which containers should be stopped (e.g. when backing up multiple volumes on different schedules), you can set the `BACKUP_STOP_DURING_BACKUP_LABEL` environment variable and then use the same value for labeling:
 
 ```yml
 version: '3'
@@ -28,7 +28,7 @@ services:
   backup:
     image: offen/docker-volume-backup:v2
     environment:
-      BACKUP_STOP_CONTAINER_LABEL: service1
+      BACKUP_STOP_DURING_BACKUP_LABEL: service1
     volumes:
       - data:/backup/my-app-backup:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro

docs/recipes/index.md

@@ -352,7 +352,7 @@ services:
       AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
       AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       # Label the container using the `data_1` volume as `docker-volume-backup.stop-during-backup=service1`
-      BACKUP_STOP_CONTAINER_LABEL: service1
+      BACKUP_STOP_DURING_BACKUP_LABEL: service1
     volumes:
       - data_1:/backup/data-1-backup:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
@@ -362,7 +362,7 @@ services:
       <<: *backup_environment
       # Label the container using the `data_2` volume as `docker-volume-backup.stop-during-backup=service2`
       BACKUP_CRON_EXPRESSION: "0 3 * * *"
-      BACKUP_STOP_CONTAINER_LABEL: service2
+      BACKUP_STOP_DURING_BACKUP_LABEL: service2
     volumes:
       - data_2:/backup/data-2-backup:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro

docs/reference/index.md

@@ -316,15 +316,22 @@ You can populate below template according to your requirements and use it as you
 
 # GPG_PASSPHRASE="<xxx>"
 
-########### STOPPING CONTAINERS DURING BACKUP
+########### STOPPING CONTAINERS AND SERVICES DURING BACKUP
 
-# Containers can be stopped by applying a
-# `docker-volume-backup.stop-during-backup` label. By default, all containers
-# that are labeled with `true` will be stopped. If you need more fine grained
-# control (e.g. when running multiple containers based on this image), you can
-# override this default by specifying a different value here.
+# Containers or services can be stopped by applying a
+# `docker-volume-backup.stop-during-backup` label. By default, all containers and
+# services that are labeled with `true` will be stopped. If you need more fine
+# grained control (e.g. when running multiple containers based on this image),
+# you can override this default by specifying a different value here.
+# BACKUP_STOP_DURING_BACKUP_LABEL="service1"
 
-# BACKUP_STOP_CONTAINER_LABEL="service1"
+# When trying to scale down Docker Swarm services, give up after
+# the specified amount of time in case the service has not converged yet.
+# In case you need to adjust this timeout, supply a duration
+# value as per https://pkg.go.dev/time#ParseDuration to `BACKUP_STOP_SERVICE_TIMEOUT`.
+# Defaults to 5 minutes.
+
+# BACKUP_STOP_SERVICE_TIMEOUT="5m"
 
 ########### EXECUTING COMMANDS IN CONTAINERS PRE/POST BACKUP
 

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/docker/cli v24.0.1+incompatible
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/gofrs/flock v0.8.1
-	github.com/klauspost/compress v1.17.4
+	github.com/klauspost/compress v1.17.5
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/offen/envconfig v1.5.0

go.sum

@@ -456,8 +456,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E=
+github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=