Periodically collect runtime info when requested

* Hoist control for exiting script a level up

* Do not accidentally nil out errors

* Log when running schedule

* Remove duplicate log line

* Warn on cron schedule that will never run

.github/workflows/deploy-docs.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:

.github/workflows/golangci-lint.yml

@@ -15,8 +15,8 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: '1.21'
           cache: false

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
       contents: read
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
   test:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2

Dockerfile

@@ -16,6 +16,5 @@ WORKDIR /root
 RUN apk add --no-cache ca-certificates
 
 COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
-COPY --chmod=755 ./entrypoint.sh /root/
 
-ENTRYPOINT ["/root/entrypoint.sh"]
+ENTRYPOINT ["/usr/bin/backup", "-foreground"]

cmd/backup/config.go

@@ -34,10 +34,13 @@ type Config struct {
 	BackupFilenameExpand          bool            `split_words:"true"`
 	BackupLatestSymlink           string          `split_words:"true"`
 	BackupArchive                 string          `split_words:"true" default:"/archive"`
+	BackupCronExpression          string          `split_words:"true" default:"@daily"`
 	BackupRetentionDays           int32           `split_words:"true" default:"-1"`
 	BackupPruningLeeway           time.Duration   `split_words:"true" default:"1m"`
 	BackupPruningPrefix           string          `split_words:"true"`
-	BackupStopContainerLabel      string          `split_words:"true" default:"true"`
+	BackupStopContainerLabel      string          `split_words:"true"`
+	BackupStopDuringBackupLabel   string          `split_words:"true" default:"true"`
+	BackupStopServiceTimeout      time.Duration   `split_words:"true" default:"5m"`
 	BackupFromSnapshot            bool            `split_words:"true"`
 	BackupExcludeRegexp           RegexpDecoder   `split_words:"true"`
 	BackupSkipBackendsFromPrune   []string        `split_words:"true"`

cmd/backup/config_provider.go

@@ -0,0 +1,87 @@
+// Copyright 2021-2022 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/joho/godotenv"
+	"github.com/offen/envconfig"
+)
+
+// envProxy is a function that mimics os.LookupEnv but can read values from any other source
+type envProxy func(string) (string, bool)
+
+func loadConfig(lookup envProxy) (*Config, error) {
+	envconfig.Lookup = func(key string) (string, bool) {
+		value, okValue := lookup(key)
+		location, okFile := lookup(key + "_FILE")
+
+		switch {
+		case okValue && !okFile: // only value
+			return value, true
+		case !okValue && okFile: // only file
+			contents, err := os.ReadFile(location)
+			if err != nil {
+				return "", false
+			}
+			return string(contents), true
+		case okValue && okFile: // both
+			return "", false
+		default: // neither, ignore
+			return "", false
+		}
+	}
+
+	var c = &Config{}
+	if err := envconfig.Process("", c); err != nil {
+		return nil, fmt.Errorf("loadConfig: failed to process configuration values: %w", err)
+	}
+
+	return c, nil
+}
+
+func loadEnvVars() (*Config, error) {
+	return loadConfig(os.LookupEnv)
+}
+
+type configFile struct {
+	name   string
+	config *Config
+}
+
+func loadEnvFiles(directory string) ([]configFile, error) {
+	items, err := os.ReadDir(directory)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, err
+		}
+		return nil, fmt.Errorf("loadEnvFiles: failed to read files from env directory: %w", err)
+	}
+
+	cs := []configFile{}
+	for _, item := range items {
+		if item.IsDir() {
+			continue
+		}
+		p := filepath.Join(directory, item.Name())
+		envFile, err := godotenv.Read(p)
+		if err != nil {
+			return nil, fmt.Errorf("loadEnvFiles: error reading config file %s: %w", p, err)
+		}
+		lookup := func(key string) (string, bool) {
+			val, ok := envFile[key]
+			return val, ok
+		}
+		c, err := loadConfig(lookup)
+		if err != nil {
+			return nil, fmt.Errorf("loadEnvFiles: error loading config from file %s: %w", p, err)
+		}
+		cs = append(cs, configFile{config: c, name: item.Name()})
+	}
+
+	return cs, nil
+}

cmd/backup/cron.go

@@ -0,0 +1,29 @@
+// Copyright 2024 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"time"
+
+	"github.com/robfig/cron/v3"
+)
+
+// checkCronSchedule detects whether the given cron expression will actually
+// ever be executed or not.
+func checkCronSchedule(expression string) (ok bool) {
+	defer func() {
+		if err := recover(); err != nil {
+			ok = false
+		}
+	}()
+	sched, err := cron.ParseStandard(expression)
+	if err != nil {
+		ok = false
+		return
+	}
+	now := time.Now()
+	sched.Next(now) // panics when the cron would never run
+	ok = true
+	return
+}

cmd/backup/exec.go

@@ -188,13 +188,18 @@ func (s *script) withLabeledCommands(step lifecyclePhase, cb func() error) func(
 	if s.cli == nil {
 		return cb
 	}
-	return func() error {
-		if err := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil {
-			return fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err)
+	return func() (err error) {
+		if err = s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil {
+			err = fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err)
+			return
 		}
 		defer func() {
-			s.must(s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step)))
+			derr := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step))
+			if err == nil && derr != nil {
+				err = derr
+			}
 		}()
-		return cb()
+		err = cb()
+		return
 	}
 }

cmd/backup/main.go

@@ -4,60 +4,246 @@
 package main
 
 import (
+	"flag"
 	"fmt"
+	"log/slog"
 	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
+
+	"github.com/robfig/cron/v3"
 )
 
-func main() {
-	s, err := newScript()
-	if err != nil {
-		panic(err)
+type command struct {
+	logger *slog.Logger
+}
+
+func newCommand() *command {
+	return &command{
+		logger: slog.New(slog.NewTextHandler(os.Stdout, nil)),
 	}
+}
 
-	unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
-	defer s.must(unlock())
-	s.must(err)
+func (c *command) must(err error) {
+	if err != nil {
+		c.logger.Error(
+			fmt.Sprintf("Fatal error running command: %v", err),
+			"error",
+			err,
+		)
+		os.Exit(1)
+	}
+}
 
+func runScript(c *Config) (err error) {
 	defer func() {
-		if pArg := recover(); pArg != nil {
-			if err, ok := pArg.(error); ok {
-				if hookErr := s.runHooks(err); hookErr != nil {
-					s.logger.Error(
-						fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr),
-					)
-				}
-				os.Exit(1)
-			}
-			panic(pArg)
+		if derr := recover(); derr != nil {
+			err = fmt.Errorf("runScript: unexpected panic running script: %v", err)
 		}
-
-		if err := s.runHooks(nil); err != nil {
-			s.logger.Error(
-				fmt.Sprintf(
-					"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
-					err,
-				),
-			)
-			os.Exit(1)
-		}
-		s.logger.Info("Finished running backup tasks.")
 	}()
 
-	s.must(s.withLabeledCommands(lifecyclePhaseArchive, func() error {
-		restartContainers, err := s.stopContainersAndServices()
-		// The mechanism for restarting containers is not using hooks as it
-		// should happen as soon as possible (i.e. before uploading backups or
-		// similar).
-		defer func() {
-			s.must(restartContainers())
-		}()
-		if err != nil {
-			return err
-		}
-		return s.createArchive()
-	})())
+	s, err := newScript(c)
+	if err != nil {
+		err = fmt.Errorf("runScript: error instantiating script: %w", err)
+		return
+	}
 
-	s.must(s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)())
-	s.must(s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)())
-	s.must(s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)())
+	runErr := func() (err error) {
+		unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
+		if err != nil {
+			err = fmt.Errorf("runScript: error acquiring file lock: %w", err)
+			return
+		}
+
+		defer func() {
+			derr := unlock()
+			if err == nil && derr != nil {
+				err = fmt.Errorf("runScript: error releasing file lock: %w", derr)
+			}
+		}()
+
+		scriptErr := func() error {
+			if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) {
+				restartContainersAndServices, err := s.stopContainersAndServices()
+				// The mechanism for restarting containers is not using hooks as it
+				// should happen as soon as possible (i.e. before uploading backups or
+				// similar).
+				defer func() {
+					derr := restartContainersAndServices()
+					if err == nil {
+						err = derr
+					}
+				}()
+				if err != nil {
+					return
+				}
+				err = s.createArchive()
+				return
+			})(); err != nil {
+				return err
+			}
+
+			if err := s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)(); err != nil {
+				return err
+			}
+			if err := s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)(); err != nil {
+				return err
+			}
+			if err := s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)(); err != nil {
+				return err
+			}
+			return nil
+		}()
+
+		if hookErr := s.runHooks(scriptErr); hookErr != nil {
+			if scriptErr != nil {
+				return fmt.Errorf(
+					"runScript: error %w executing the script followed by %w calling the registered hooks",
+					scriptErr,
+					hookErr,
+				)
+			}
+			return fmt.Errorf(
+				"runScript: the script ran successfully, but an error occurred calling the registered hooks: %w",
+				hookErr,
+			)
+		}
+		if scriptErr != nil {
+			return fmt.Errorf("runScript: error running script: %w", scriptErr)
+		}
+		return nil
+	}()
+
+	if runErr != nil {
+		s.logger.Error(
+			fmt.Sprintf("Script run failed: %v", runErr), "error", runErr,
+		)
+	}
+	return runErr
+}
+
+func (c *command) runInForeground(profileCronExpression string) error {
+	cr := cron.New(
+		cron.WithParser(
+			cron.NewParser(
+				cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
+			),
+		),
+	)
+
+	addJob := func(config *Config, name string) error {
+		if _, err := cr.AddFunc(config.BackupCronExpression, func() {
+			c.logger.Info(
+				fmt.Sprintf(
+					"Now running script on schedule %s",
+					config.BackupCronExpression,
+				),
+			)
+			if err := runScript(config); err != nil {
+				c.logger.Error(
+					fmt.Sprintf(
+						"Unexpected error running schedule %s: %v",
+						config.BackupCronExpression,
+						err,
+					),
+					"error",
+					err,
+				)
+			}
+		}); err != nil {
+			return fmt.Errorf("addJob: error adding schedule %s: %w", config.BackupCronExpression, err)
+		}
+
+		c.logger.Info(fmt.Sprintf("Successfully scheduled backup %s with expression %s", name, config.BackupCronExpression))
+		if ok := checkCronSchedule(config.BackupCronExpression); !ok {
+			c.logger.Warn(
+				fmt.Sprintf("Scheduled cron expression %s will never run, is this intentional?", config.BackupCronExpression),
+			)
+		}
+
+		return nil
+	}
+
+	cs, err := loadEnvFiles("/etc/dockervolumebackup/conf.d")
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return fmt.Errorf("runInForeground: could not load config from environment files: %w", err)
+		}
+
+		c, err := loadEnvVars()
+		if err != nil {
+			return fmt.Errorf("runInForeground: could not load config from environment variables: %w", err)
+		} else {
+			err = addJob(c, "from environment")
+			if err != nil {
+				return fmt.Errorf("runInForeground: error adding job from env: %w", err)
+			}
+		}
+	} else {
+		c.logger.Info("/etc/dockervolumebackup/conf.d was found, using configuration files from this directory.")
+		for _, config := range cs {
+			err = addJob(config.config, config.name)
+			if err != nil {
+				return fmt.Errorf("runInForeground: error adding jobs from conf files: %w", err)
+			}
+		}
+	}
+
+	if profileCronExpression != "" {
+		if _, err := cr.AddFunc(profileCronExpression, func() {
+			memStats := runtime.MemStats{}
+			runtime.ReadMemStats(&memStats)
+			c.logger.Info(
+				"Collecting runtime information",
+				"num_goroutines",
+				runtime.NumGoroutine(),
+				"memory_heap_alloc",
+				formatBytes(memStats.HeapAlloc, false),
+				"memory_heap_inuse",
+				formatBytes(memStats.HeapInuse, false),
+				"memory_heap_sys",
+				formatBytes(memStats.HeapSys, false),
+				"memory_heap_objects",
+				memStats.HeapObjects,
+			)
+		}); err != nil {
+			return fmt.Errorf("runInForeground: error adding profiling job: %w", err)
+		}
+	}
+
+	var quit = make(chan os.Signal, 1)
+	signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT)
+	cr.Start()
+	<-quit
+	ctx := cr.Stop()
+	<-ctx.Done()
+
+	return nil
+}
+
+func (c *command) runAsCommand() error {
+	config, err := loadEnvVars()
+	if err != nil {
+		return fmt.Errorf("runAsCommand: error loading env vars: %w", err)
+	}
+	err = runScript(config)
+	if err != nil {
+		return fmt.Errorf("runAsCommand: error running script: %w", err)
+	}
+
+	return nil
+}
+
+func main() {
+	foreground := flag.Bool("foreground", false, "run the tool in the foreground")
+	profile := flag.String("profile", "", "collect runtime metrics and log them periodically on the given cron expression")
+	flag.Parse()
+
+	c := newCommand()
+	if *foreground {
+		c.must(c.runInForeground(*profile))
+	} else {
+		c.must(c.runAsCommand())
+	}
 }

cmd/backup/script.go

@@ -5,8 +5,6 @@ package main
 
 import (
 	"bytes"
-	"context"
-	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -30,14 +28,8 @@ import (
 	openpgp "github.com/ProtonMail/go-crypto/openpgp/v2"
 	"github.com/containrrr/shoutrrr"
 	"github.com/containrrr/shoutrrr/pkg/router"
-	"github.com/docker/cli/cli/command/service/progress"
-	"github.com/docker/docker/api/types"
-	ctr "github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/client"
 	"github.com/leekchan/timeutil"
-	"github.com/offen/envconfig"
 	"github.com/otiai10/copy"
 	"golang.org/x/sync/errgroup"
 )
@@ -65,10 +57,10 @@ type script struct {
 // remote resources like the Docker engine or remote storage locations. All
 // reading from env vars or other configuration sources is expected to happen
 // in this method.
-func newScript() (*script, error) {
+func newScript(c *Config) (*script, error) {
 	stdOut, logBuffer := buffer(os.Stdout)
 	s := &script{
-		c:      &Config{},
+		c:      c,
 		logger: slog.New(slog.NewTextHandler(stdOut, nil)),
 		stats: &Stats{
 			StartTime: time.Now(),
@@ -90,32 +82,6 @@ func newScript() (*script, error) {
 		return nil
 	})
 
-	envconfig.Lookup = func(key string) (string, bool) {
-		value, okValue := os.LookupEnv(key)
-		location, okFile := os.LookupEnv(key + "_FILE")
-
-		switch {
-		case okValue && !okFile: // only value
-			return value, true
-		case !okValue && okFile: // only file
-			contents, err := os.ReadFile(location)
-			if err != nil {
-				s.must(fmt.Errorf("newScript: failed to read %s! Error: %s", location, err))
-				return "", false
-			}
-			return string(contents), true
-		case okValue && okFile: // both
-			s.must(fmt.Errorf("newScript: both %s and %s are set!", key, key+"_FILE"))
-			return "", false
-		default: // neither, ignore
-			return "", false
-		}
-	}
-
-	if err := envconfig.Process("", s.c); err != nil {
-		return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
-	}
-
 	s.file = path.Join("/tmp", s.c.BackupFilename)
 
 	tmplFileName, tErr := template.New("extension").Parse(s.file)
@@ -146,6 +112,12 @@ func newScript() (*script, error) {
 			return nil, fmt.Errorf("newScript: failed to create docker client")
 		}
 		s.cli = cli
+		s.registerHook(hookLevelPlumbing, func(err error) error {
+			if err := s.cli.Close(); err != nil {
+				return fmt.Errorf("newScript: failed to close docker client: %w", err)
+			}
+			return nil
+		})
 	}
 
 	logFunc := func(logType storage.LogLevel, context string, msg string, params ...any) {
@@ -319,302 +291,6 @@ func newScript() (*script, error) {
 	return s, nil
 }
 
-type noopWriteCloser struct {
-	io.Writer
-}
-
-func (noopWriteCloser) Close() error {
-	return nil
-}
-
-type handledSwarmService struct {
-	serviceID           string
-	initialReplicaCount uint64
-}
-
-// stopContainersAndServices stops all Docker containers that are marked as to being
-// stopped during the backup and returns a function that can be called to
-// restart everything that has been stopped.
-func (s *script) stopContainersAndServices() (func() error, error) {
-	if s.cli == nil {
-		return noop, nil
-	}
-
-	dockerInfo, err := s.cli.Info(context.Background())
-	if err != nil {
-		return noop, fmt.Errorf("(*script).stopContainersAndServices: error getting docker info: %w", err)
-	}
-	isDockerSwarm := dockerInfo.Swarm.LocalNodeState != "inactive"
-	discardWriter := &noopWriteCloser{io.Discard}
-
-	filterMatchLabel := fmt.Sprintf(
-		"docker-volume-backup.stop-during-backup=%s",
-		s.c.BackupStopContainerLabel,
-	)
-
-	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{})
-	if err != nil {
-		return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for containers: %w", err)
-	}
-	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Filters: filters.NewArgs(filters.KeyValuePair{
-			Key:   "label",
-			Value: filterMatchLabel,
-		}),
-	})
-	if err != nil {
-		return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for containers to stop: %w", err)
-	}
-
-	var allServices []swarm.Service
-	var servicesToScaleDown []handledSwarmService
-	if isDockerSwarm {
-		allServices, err = s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
-		if err != nil {
-			return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for services: %w", err)
-		}
-		matchingServices, err := s.cli.ServiceList(context.Background(), types.ServiceListOptions{
-			Filters: filters.NewArgs(filters.KeyValuePair{
-				Key:   "label",
-				Value: filterMatchLabel,
-			}),
-			Status: true,
-		})
-		for _, s := range matchingServices {
-			servicesToScaleDown = append(servicesToScaleDown, handledSwarmService{
-				serviceID:           s.ID,
-				initialReplicaCount: *s.Spec.Mode.Replicated.Replicas,
-			})
-		}
-		if err != nil {
-			return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for services to scale down: %w", err)
-		}
-	}
-
-	if len(containersToStop) == 0 && len(servicesToScaleDown) == 0 {
-		return noop, nil
-	}
-
-	if isDockerSwarm {
-		for _, container := range containersToStop {
-			if swarmServiceID, ok := container.Labels["com.docker.swarm.service.id"]; ok {
-				parentService, _, err := s.cli.ServiceInspectWithRaw(context.Background(), swarmServiceID, types.ServiceInspectOptions{})
-				if err != nil {
-					return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for parent service with ID %s: %w", swarmServiceID, err)
-				}
-				for label := range parentService.Spec.Labels {
-					if label == "docker-volume-backup.stop-during-backup" {
-						return noop, fmt.Errorf(
-							"(*script).stopContainersAndServices: container %s is labeled to stop but has parent service %s which is also labeled, cannot continue",
-							container.Names[0],
-							parentService.Spec.Name,
-						)
-					}
-				}
-			}
-		}
-	}
-
-	s.logger.Info(
-		fmt.Sprintf(
-			"Stopping %d out of %d running container(s) and scaling down %d out of %d active service(s) as they were labeled %s.",
-			len(containersToStop),
-			len(allContainers),
-			len(servicesToScaleDown),
-			len(allServices),
-			filterMatchLabel,
-		),
-	)
-
-	var stoppedContainers []types.Container
-	var stopErrors []error
-	for _, container := range containersToStop {
-		if err := s.cli.ContainerStop(context.Background(), container.ID, ctr.StopOptions{}); err != nil {
-			stopErrors = append(stopErrors, err)
-		} else {
-			stoppedContainers = append(stoppedContainers, container)
-		}
-	}
-
-	var scaledDownServices []swarm.Service
-	var scaleDownErrors []error
-	if isDockerSwarm {
-		for _, svc := range servicesToScaleDown {
-			service, _, err := s.cli.ServiceInspectWithRaw(context.Background(), svc.serviceID, types.ServiceInspectOptions{})
-			if err != nil {
-				scaleDownErrors = append(
-					scaleDownErrors,
-					fmt.Errorf("(*script).stopContainersAndServices: error inspecting service %s: %w", svc.serviceID, err),
-				)
-				continue
-			}
-			var zero uint64 = 0
-			serviceMode := &service.Spec.Mode
-			switch {
-			case serviceMode.Replicated != nil:
-				serviceMode.Replicated.Replicas = &zero
-			default:
-				scaleDownErrors = append(
-					scaleDownErrors,
-					fmt.Errorf("(*script).stopContainersAndServices: labeled service %s has to be in replicated mode", service.Spec.Name),
-				)
-				continue
-			}
-
-			response, err := s.cli.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{})
-			if err != nil {
-				scaleDownErrors = append(scaleDownErrors, err)
-				continue
-			}
-
-			for _, warning := range response.Warnings {
-				s.logger.Warn(
-					fmt.Sprintf("The Docker API returned a warning when scaling down service %s: %s", service.Spec.Name, warning),
-				)
-			}
-
-			if err := progress.ServiceProgress(context.Background(), s.cli, service.ID, discardWriter); err != nil {
-				scaleDownErrors = append(scaleDownErrors, err)
-			} else {
-				scaledDownServices = append(scaledDownServices, service)
-			}
-
-			// progress.ServiceProgress returns too early, so we need to manually check
-			// whether all containers belonging to the service have actually been removed
-			for {
-				containers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-					Filters: filters.NewArgs(filters.KeyValuePair{
-						Key:   "label",
-						Value: fmt.Sprintf("com.docker.swarm.service.id=%s", service.ID),
-					}),
-				})
-				if err != nil {
-					scaleDownErrors = append(scaleDownErrors, err)
-					break
-				}
-				if len(containers) == 0 {
-					break
-				}
-				time.Sleep(time.Second)
-			}
-		}
-	}
-
-	s.stats.Containers = ContainersStats{
-		All:        uint(len(allContainers)),
-		ToStop:     uint(len(containersToStop)),
-		Stopped:    uint(len(stoppedContainers)),
-		StopErrors: uint(len(stopErrors)),
-	}
-
-	s.stats.Services = ServicesStats{
-		All:             uint(len(allServices)),
-		ToScaleDown:     uint(len(servicesToScaleDown)),
-		ScaledDown:      uint(len(scaledDownServices)),
-		ScaleDownErrors: uint(len(scaleDownErrors)),
-	}
-
-	var initialErr error
-	allErrors := append(stopErrors, scaleDownErrors...)
-	if len(allErrors) != 0 {
-		initialErr = fmt.Errorf(
-			"(*script).stopContainersAndServices: %d error(s) stopping containers: %w",
-			len(allErrors),
-			errors.Join(allErrors...),
-		)
-	}
-
-	return func() error {
-		servicesRequiringForceUpdate := map[string]struct{}{}
-
-		var restartErrors []error
-		for _, container := range stoppedContainers {
-			if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok {
-				servicesRequiringForceUpdate[swarmServiceName] = struct{}{}
-				continue
-			}
-			if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil {
-				restartErrors = append(restartErrors, err)
-			}
-		}
-
-		if len(servicesRequiringForceUpdate) != 0 {
-			services, _ := s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
-			for serviceName := range servicesRequiringForceUpdate {
-				var serviceMatch swarm.Service
-				for _, service := range services {
-					if service.Spec.Name == serviceName {
-						serviceMatch = service
-						break
-					}
-				}
-				if serviceMatch.ID == "" {
-					restartErrors = append(
-						restartErrors,
-						fmt.Errorf("(*script).stopContainersAndServices: couldn't find service with name %s", serviceName),
-					)
-					continue
-				}
-				serviceMatch.Spec.TaskTemplate.ForceUpdate += 1
-				if _, err := s.cli.ServiceUpdate(
-					context.Background(), serviceMatch.ID,
-					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},
-				); err != nil {
-					restartErrors = append(restartErrors, err)
-				}
-			}
-		}
-
-		var scaleUpErrors []error
-		if isDockerSwarm {
-			for _, svc := range servicesToScaleDown {
-				service, _, err := s.cli.ServiceInspectWithRaw(context.Background(), svc.serviceID, types.ServiceInspectOptions{})
-				if err != nil {
-					scaleUpErrors = append(scaleUpErrors, err)
-					continue
-				}
-
-				service.Spec.Mode.Replicated.Replicas = &svc.initialReplicaCount
-				response, err := s.cli.ServiceUpdate(
-					context.Background(),
-					service.ID,
-					service.Version, service.Spec,
-					types.ServiceUpdateOptions{},
-				)
-				if err != nil {
-					scaleUpErrors = append(scaleUpErrors, err)
-					continue
-				}
-				for _, warning := range response.Warnings {
-					s.logger.Warn(
-						fmt.Sprintf("The Docker API returned a warning when scaling up service %s: %s", service.Spec.Name, warning),
-					)
-				}
-				if err := progress.ServiceProgress(context.Background(), s.cli, service.ID, discardWriter); err != nil {
-					scaleUpErrors = append(scaleUpErrors, err)
-				}
-			}
-		}
-
-		allErrors := append(restartErrors, scaleUpErrors...)
-		if len(allErrors) != 0 {
-			return fmt.Errorf(
-				"stopContainers: %d error(s) restarting containers and services: %w",
-				len(allErrors),
-				errors.Join(allErrors...),
-			)
-		}
-		s.logger.Info(
-			fmt.Sprintf(
-				"Restarted %d container(s) and %d service(s).",
-				len(stoppedContainers),
-				len(scaledDownServices),
-			),
-		)
-		return nil
-	}, initialErr
-}
-
 // createArchive creates a tar archive of the configured backup location and
 // saves it to disk.
 func (s *script) createArchive() error {
@@ -625,7 +301,7 @@ func (s *script) createArchive() error {
 			"Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.",
 		)
 		s.logger.Warn(
-			"Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the README for an upgrade guide.",
+			"Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the documentation for an upgrade guide.",
 		)
 		backupSources = filepath.Join("/tmp", s.c.BackupSources)
 		// copy before compressing guard against a situation where backup folder's content are still growing.
@@ -810,17 +486,6 @@ func (s *script) pruneBackups() error {
 	return nil
 }
 
-// must exits the script run prematurely in case the given error
-// is non-nil.
-func (s *script) must(err error) {
-	if err != nil {
-		s.logger.Error(
-			fmt.Sprintf("Fatal error running backup: %s", err),
-		)
-		panic(err)
-	}
-}
-
 // skipPrune returns true if the given backend name is contained in the
 // list of skipped backends.
 func skipPrune(name string, skippedBackends []string) bool {

cmd/backup/stop_restart.go

@@ -0,0 +1,338 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/docker/cli/cli/command/service/progress"
+	"github.com/docker/docker/api/types"
+	ctr "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+)
+
+func scaleService(cli *client.Client, serviceID string, replicas uint64) ([]string, error) {
+	service, _, err := cli.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("scaleService: error inspecting service %s: %w", serviceID, err)
+	}
+	serviceMode := &service.Spec.Mode
+	switch {
+	case serviceMode.Replicated != nil:
+		serviceMode.Replicated.Replicas = &replicas
+	default:
+		return nil, fmt.Errorf("scaleService: service to be scaled %s has to be in replicated mode", service.Spec.Name)
+	}
+
+	response, err := cli.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("scaleService: error updating service: %w", err)
+	}
+
+	discardWriter := &noopWriteCloser{io.Discard}
+	if err := progress.ServiceProgress(context.Background(), cli, service.ID, discardWriter); err != nil {
+		return nil, err
+	}
+	return response.Warnings, nil
+}
+
+func awaitContainerCountForService(cli *client.Client, serviceID string, count int, timeoutAfter time.Duration) error {
+	poll := time.NewTicker(time.Second)
+	timeout := time.NewTimer(timeoutAfter)
+	defer timeout.Stop()
+	defer poll.Stop()
+
+	for {
+		select {
+		case <-timeout.C:
+			return fmt.Errorf(
+				"awaitContainerCount: timed out after waiting %s for service %s to reach desired container count of %d",
+				timeoutAfter,
+				serviceID,
+				count,
+			)
+		case <-poll.C:
+			containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{
+				Filters: filters.NewArgs(filters.KeyValuePair{
+					Key:   "label",
+					Value: fmt.Sprintf("com.docker.swarm.service.id=%s", serviceID),
+				}),
+			})
+			if err != nil {
+				return fmt.Errorf("awaitContainerCount: error listing containers: %w", err)
+			}
+			if len(containers) == count {
+				return nil
+			}
+		}
+	}
+}
+
+// stopContainersAndServices stops all Docker containers that are marked as to being
+// stopped during the backup and returns a function that can be called to
+// restart everything that has been stopped.
+func (s *script) stopContainersAndServices() (func() error, error) {
+	if s.cli == nil {
+		return noop, nil
+	}
+
+	dockerInfo, err := s.cli.Info(context.Background())
+	if err != nil {
+		return noop, fmt.Errorf("(*script).stopContainersAndServices: error getting docker info: %w", err)
+	}
+	isDockerSwarm := dockerInfo.Swarm.LocalNodeState != "inactive"
+
+	labelValue := s.c.BackupStopDuringBackupLabel
+	if s.c.BackupStopContainerLabel != "" {
+		s.logger.Warn(
+			"Using BACKUP_STOP_CONTAINER_LABEL has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use BACKUP_STOP_DURING_BACKUP_LABEL instead. Refer to the docs for an upgrade guide.",
+		)
+		if _, ok := os.LookupEnv("BACKUP_STOP_DURING_BACKUP_LABEL"); ok {
+			return noop, errors.New("(*script).stopContainersAndServices: both BACKUP_STOP_DURING_BACKUP_LABEL and BACKUP_STOP_CONTAINER_LABEL have been set, cannot continue")
+		}
+		labelValue = s.c.BackupStopContainerLabel
+	}
+
+	filterMatchLabel := fmt.Sprintf(
+		"docker-volume-backup.stop-during-backup=%s",
+		labelValue,
+	)
+
+	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{})
+	if err != nil {
+		return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for containers: %w", err)
+	}
+	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
+		Filters: filters.NewArgs(filters.KeyValuePair{
+			Key:   "label",
+			Value: filterMatchLabel,
+		}),
+	})
+	if err != nil {
+		return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for containers to stop: %w", err)
+	}
+
+	var allServices []swarm.Service
+	var servicesToScaleDown []handledSwarmService
+	if isDockerSwarm {
+		allServices, err = s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
+		if err != nil {
+			return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for services: %w", err)
+		}
+		matchingServices, err := s.cli.ServiceList(context.Background(), types.ServiceListOptions{
+			Filters: filters.NewArgs(filters.KeyValuePair{
+				Key:   "label",
+				Value: filterMatchLabel,
+			}),
+			Status: true,
+		})
+		for _, s := range matchingServices {
+			servicesToScaleDown = append(servicesToScaleDown, handledSwarmService{
+				serviceID:           s.ID,
+				initialReplicaCount: *s.Spec.Mode.Replicated.Replicas,
+			})
+		}
+		if err != nil {
+			return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for services to scale down: %w", err)
+		}
+	}
+
+	if len(containersToStop) == 0 && len(servicesToScaleDown) == 0 {
+		return noop, nil
+	}
+
+	if isDockerSwarm {
+		for _, container := range containersToStop {
+			if swarmServiceID, ok := container.Labels["com.docker.swarm.service.id"]; ok {
+				parentService, _, err := s.cli.ServiceInspectWithRaw(context.Background(), swarmServiceID, types.ServiceInspectOptions{})
+				if err != nil {
+					return noop, fmt.Errorf("(*script).stopContainersAndServices: error querying for parent service with ID %s: %w", swarmServiceID, err)
+				}
+				for label := range parentService.Spec.Labels {
+					if label == "docker-volume-backup.stop-during-backup" {
+						return noop, fmt.Errorf(
+							"(*script).stopContainersAndServices: container %s is labeled to stop but has parent service %s which is also labeled, cannot continue",
+							container.Names[0],
+							parentService.Spec.Name,
+						)
+					}
+				}
+			}
+		}
+	}
+
+	s.logger.Info(
+		fmt.Sprintf(
+			"Stopping %d out of %d running container(s) as they were labeled %s.",
+			len(containersToStop),
+			len(allContainers),
+			filterMatchLabel,
+		),
+	)
+	if isDockerSwarm {
+		s.logger.Info(
+			fmt.Sprintf(
+				"Scaling down %d out of %d active service(s) as they were labeled %s.",
+				len(servicesToScaleDown),
+				len(allServices),
+				filterMatchLabel,
+			),
+		)
+	}
+
+	var stoppedContainers []types.Container
+	var stopErrors []error
+	for _, container := range containersToStop {
+		if err := s.cli.ContainerStop(context.Background(), container.ID, ctr.StopOptions{}); err != nil {
+			stopErrors = append(stopErrors, err)
+		} else {
+			stoppedContainers = append(stoppedContainers, container)
+		}
+	}
+
+	var scaledDownServices []handledSwarmService
+	var scaleDownErrors concurrentSlice[error]
+	if isDockerSwarm {
+		wg := sync.WaitGroup{}
+		for _, svc := range servicesToScaleDown {
+			wg.Add(1)
+			go func(svc handledSwarmService) {
+				defer wg.Done()
+				warnings, err := scaleService(s.cli, svc.serviceID, 0)
+				if err != nil {
+					scaleDownErrors.append(err)
+					return
+				}
+				scaledDownServices = append(scaledDownServices, svc)
+				for _, warning := range warnings {
+					s.logger.Warn(
+						fmt.Sprintf("The Docker API returned a warning when scaling down service %s: %s", svc.serviceID, warning),
+					)
+				}
+				// progress.ServiceProgress returns too early, so we need to manually check
+				// whether all containers belonging to the service have actually been removed
+				if err := awaitContainerCountForService(s.cli, svc.serviceID, 0, s.c.BackupStopServiceTimeout); err != nil {
+					scaleDownErrors.append(err)
+				}
+			}(svc)
+		}
+		wg.Wait()
+	}
+
+	s.stats.Containers = ContainersStats{
+		All:        uint(len(allContainers)),
+		ToStop:     uint(len(containersToStop)),
+		Stopped:    uint(len(stoppedContainers)),
+		StopErrors: uint(len(stopErrors)),
+	}
+
+	s.stats.Services = ServicesStats{
+		All:             uint(len(allServices)),
+		ToScaleDown:     uint(len(servicesToScaleDown)),
+		ScaledDown:      uint(len(scaledDownServices)),
+		ScaleDownErrors: uint(len(scaleDownErrors.value())),
+	}
+
+	var initialErr error
+	allErrors := append(stopErrors, scaleDownErrors.value()...)
+	if len(allErrors) != 0 {
+		initialErr = fmt.Errorf(
+			"(*script).stopContainersAndServices: %d error(s) stopping containers: %w",
+			len(allErrors),
+			errors.Join(allErrors...),
+		)
+	}
+
+	return func() error {
+		var restartErrors []error
+		matchedServices := map[string]bool{}
+		for _, container := range stoppedContainers {
+			if swarmServiceID, ok := container.Labels["com.docker.swarm.service.id"]; ok && isDockerSwarm {
+				if _, ok := matchedServices[swarmServiceID]; ok {
+					continue
+				}
+				matchedServices[swarmServiceID] = true
+				// in case a container was part of a swarm service, the service requires to
+				// be force updated instead of restarting the container as it would otherwise
+				// remain in a "completed" state
+				service, _, err := s.cli.ServiceInspectWithRaw(context.Background(), swarmServiceID, types.ServiceInspectOptions{})
+				if err != nil {
+					restartErrors = append(
+						restartErrors,
+						fmt.Errorf("(*script).stopContainersAndServices: error looking up parent service: %w", err),
+					)
+					continue
+				}
+				service.Spec.TaskTemplate.ForceUpdate += 1
+				if _, err := s.cli.ServiceUpdate(
+					context.Background(), service.ID,
+					service.Version, service.Spec, types.ServiceUpdateOptions{},
+				); err != nil {
+					restartErrors = append(restartErrors, err)
+				}
+				continue
+			}
+
+			if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil {
+				restartErrors = append(restartErrors, err)
+			}
+		}
+
+		var scaleUpErrors concurrentSlice[error]
+		if isDockerSwarm {
+			wg := &sync.WaitGroup{}
+			for _, svc := range servicesToScaleDown {
+				wg.Add(1)
+				go func(svc handledSwarmService) {
+					defer wg.Done()
+					warnings, err := scaleService(s.cli, svc.serviceID, svc.initialReplicaCount)
+					if err != nil {
+						scaleDownErrors.append(err)
+						return
+					}
+					for _, warning := range warnings {
+						s.logger.Warn(
+							fmt.Sprintf("The Docker API returned a warning when scaling up service %s: %s", svc.serviceID, warning),
+						)
+					}
+				}(svc)
+			}
+			wg.Wait()
+		}
+
+		allErrors := append(restartErrors, scaleUpErrors.value()...)
+		if len(allErrors) != 0 {
+			return fmt.Errorf(
+				"(*script).stopContainersAndServices: %d error(s) restarting containers and services: %w",
+				len(allErrors),
+				errors.Join(allErrors...),
+			)
+		}
+
+		s.logger.Info(
+			fmt.Sprintf(
+				"Restarted %d container(s).",
+				len(stoppedContainers),
+			),
+		)
+		if isDockerSwarm {
+			s.logger.Info(
+				fmt.Sprintf(
+					"Scaled %d service(s) back up.",
+					len(scaledDownServices),
+				),
+			)
+		}
+
+		return nil
+	}, initialErr
+}

cmd/backup/util.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sync"
 )
 
 var noop = func() error { return nil }
@@ -50,3 +51,31 @@ func (b *bufferingWriter) Write(p []byte) (n int, err error) {
 	}
 	return b.writer.Write(p)
 }
+
+type noopWriteCloser struct {
+	io.Writer
+}
+
+func (noopWriteCloser) Close() error {
+	return nil
+}
+
+type handledSwarmService struct {
+	serviceID           string
+	initialReplicaCount uint64
+}
+
+type concurrentSlice[T any] struct {
+	val []T
+	sync.Mutex
+}
+
+func (c *concurrentSlice[T]) append(v T) {
+	c.Lock()
+	defer c.Unlock()
+	c.val = append(c.val, v)
+}
+
+func (c *concurrentSlice[T]) value() []T {
+	return c.val
+}

docs/how-tos/replace-deprecated-backup-stop-container-label.md

@@ -0,0 +1,19 @@
+---
+title: Replace deprecated BACKUP_STOP_CONTAINER_LABEL setting
+layout: default
+parent: How Tos
+nav_order: 19
+---
+
+# Replace deprecated `BACKUP_STOP_CONTAINER_LABEL` setting
+
+Version `v2.36.0` deprecated the `BACKUP_STOP_CONTAINER_LABEL` setting and renamed it `BACKUP_STOP_DURING_BACKUP_LABEL` which is supposed to signal that this will stop both containers _and_ services.
+Migrating is done by renaming the key for your custom value:
+
+```diff
+    env:
+-     BACKUP_STOP_CONTAINER_LABEL: database
++     BACKUP_STOP_DURING_BACKUP_LABEL: database
+```
+
+The old key will stay supported until the next major version, but logs a warning each time a backup is taken.

docs/how-tos/set-up-notifications.md

@@ -76,7 +76,7 @@ Configuration, data about the backup run and helper functions will be passed to
 
 Here is a list of all data passed to the template:
 
-* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`)
+* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_DURING_BACKUP_LABEL` becomes `BackupStopDuringBackupLabel`)
 * `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates
 * `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available.
   * `StartTime`: time when the script started execution

docs/how-tos/stop-containers-during-backup.md

@@ -14,7 +14,7 @@ In many cases, it will be desirable to stop the services that are consuming the
 This image can automatically stop and restart containers and services.
 By default, any container that is labeled `docker-volume-backup.stop-during-backup=true` will be stopped before the backup is being taken and restarted once it has finished.
 
-In case you need more fine grained control about which containers should be stopped (e.g. when backing up multiple volumes on different schedules), you can set the `BACKUP_STOP_CONTAINER_LABEL` environment variable and then use the same value for labeling:
+In case you need more fine grained control about which containers should be stopped (e.g. when backing up multiple volumes on different schedules), you can set the `BACKUP_STOP_DURING_BACKUP_LABEL` environment variable and then use the same value for labeling:
 
 ```yml
 version: '3'
@@ -28,7 +28,7 @@ services:
   backup:
     image: offen/docker-volume-backup:v2
     environment:
-      BACKUP_STOP_CONTAINER_LABEL: service1
+      BACKUP_STOP_DURING_BACKUP_LABEL: service1
     volumes:
       - data:/backup/my-app-backup:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro

docs/how-tos/use-custom-docker-host.md

@@ -13,5 +13,33 @@ If you are interfacing with Docker via TCP, set `DOCKER_HOST` to the correct URL
 DOCKER_HOST=tcp://docker_socket_proxy:2375
 ```
 
-In case you are using a socket proxy, it must support `GET` and `POST` requests to the `/containers` endpoint. If you are using Docker Swarm, it must also support the `/services` endpoint. If you are using pre/post backup commands, it must also support the `/exec` endpoint.
+If you do this as you seek to restrict access to the Docker socket, this tool is potentially calling the following Docker APIs:
 
+| API | When |
+|-|-|
+| `Info` | always |
+| `ContainerExecCreate` | running commands from `exec-labels` |
+| `ContainerExecAttach` | running commands from `exec-labels` |
+| `ContainerExecInspect` | running commands from `exec-labels` |
+| `ContainerList` | always |
+  `ServiceList` | Docker engine is running in Swarm mode |
+| `ServiceInspect` | Docker engine is running in Swarm mode |
+| `ServiceUpdate` | Docker engine is running in Swarm mode and `stop-during-backup` is used |
+| `ConatinerStop` | `stop-during-backup` labels are applied to containers |
+| `ContainerStart` | `stop-during-backup` labels are applied to container |
+
+---
+
+In case you are using [`docker-socket-proxy`][proxy], this means following permissions are required:
+
+| Permission | When |
+|-|-|
+| INFO | always required |
+| CONTAINERS | always required |
+| POST | required when using `stop-during-backup` or `exec` labels |
+| EXEC | required when using `exec`-labeled commands |
+| SERVICES | required when Docker Engine is running in Swarm mode |
+| NODES | required when labeling services `stop-during-backup` |
+| TASKS | required when labeling services `stop-during-backup` |
+
+[proxy]: https://github.com/Tecnativa/docker-socket-proxy

docs/recipes/index.md

@@ -352,7 +352,7 @@ services:
       AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
       AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       # Label the container using the `data_1` volume as `docker-volume-backup.stop-during-backup=service1`
-      BACKUP_STOP_CONTAINER_LABEL: service1
+      BACKUP_STOP_DURING_BACKUP_LABEL: service1
     volumes:
       - data_1:/backup/data-1-backup:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
@@ -362,7 +362,7 @@ services:
       <<: *backup_environment
       # Label the container using the `data_2` volume as `docker-volume-backup.stop-during-backup=service2`
       BACKUP_CRON_EXPRESSION: "0 3 * * *"
-      BACKUP_STOP_CONTAINER_LABEL: service2
+      BACKUP_STOP_DURING_BACKUP_LABEL: service2
     volumes:
       - data_2:/backup/data-2-backup:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro

docs/reference/index.md

@@ -23,9 +23,22 @@ You can populate below template according to your requirements and use it as you
 ```
 ########### BACKUP SCHEDULE
 
-# Backups run on the given cron schedule in `busybox` flavor. If no
-# value is set, `@daily` will be used. If you do not want the cron
-# to ever run, use `0 0 5 31 2 ?`.
+
+# A cron expression represents a set of times, using 5 or 6 space-separated fields.
+#
+# Field name   | Mandatory? | Allowed values  | Allowed special characters
+# ----------   | ---------- | --------------  | --------------------------
+# Seconds      | No         | 0-59            | * / , -
+# Minutes      | Yes        | 0-59            | * / , -
+# Hours        | Yes        | 0-23            | * / , -
+# Day of month | Yes        | 1-31            | * / , - ?
+# Month        | Yes        | 1-12 or JAN-DEC | * / , -
+# Day of week  | Yes        | 0-6 or SUN-SAT  | * / , - ?
+#
+# Month and Day-of-week field values are case insensitive.
+# "SUN", "Sun", and "sun" are equally accepted.
+# If no value is set, `@daily` will be used.
+# If you do not want the cron to ever run, use `0 0 5 31 2 ?`.
 
 # BACKUP_CRON_EXPRESSION="0 2 * * *"
 
@@ -316,15 +329,22 @@ You can populate below template according to your requirements and use it as you
 
 # GPG_PASSPHRASE="<xxx>"
 
-########### STOPPING CONTAINERS DURING BACKUP
+########### STOPPING CONTAINERS AND SERVICES DURING BACKUP
 
-# Containers can be stopped by applying a
-# `docker-volume-backup.stop-during-backup` label. By default, all containers
-# that are labeled with `true` will be stopped. If you need more fine grained
-# control (e.g. when running multiple containers based on this image), you can
-# override this default by specifying a different value here.
+# Containers or services can be stopped by applying a
+# `docker-volume-backup.stop-during-backup` label. By default, all containers and
+# services that are labeled with `true` will be stopped. If you need more fine
+# grained control (e.g. when running multiple containers based on this image),
+# you can override this default by specifying a different value here.
+# BACKUP_STOP_DURING_BACKUP_LABEL="service1"
 
-# BACKUP_STOP_CONTAINER_LABEL="service1"
+# When trying to scale down Docker Swarm services, give up after
+# the specified amount of time in case the service has not converged yet.
+# In case you need to adjust this timeout, supply a duration
+# value as per https://pkg.go.dev/time#ParseDuration to `BACKUP_STOP_SERVICE_TIMEOUT`.
+# Defaults to 5 minutes.
+
+# BACKUP_STOP_SERVICE_TIMEOUT="5m"
 
 ########### EXECUTING COMMANDS IN CONTAINERS PRE/POST BACKUP
 

entrypoint.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# Copyright 2021 - Offen Authors <hioffen@posteo.de>
-# SPDX-License-Identifier: MPL-2.0
-
-set -e
-
-if [ ! -d "/etc/dockervolumebackup/conf.d" ]; then
-  BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
-
-  echo "Installing cron.d entry with expression $BACKUP_CRON_EXPRESSION."
-  echo "$BACKUP_CRON_EXPRESSION backup 2>&1" | crontab -
-else
-  echo "/etc/dockervolumebackup/conf.d was found, using configuration files from this directory."
-
-  crontab -r && crontab /dev/null
-  for file in /etc/dockervolumebackup/conf.d/*; do
-    source $file
-    BACKUP_CRON_EXPRESSION="${BACKUP_CRON_EXPRESSION:-@daily}"
-    echo "Appending cron.d entry with expression $BACKUP_CRON_EXPRESSION and configuration file $file"
-    (crontab -l; echo "$BACKUP_CRON_EXPRESSION /bin/sh -c 'set -a; source $file; set +a && backup' 2>&1") | crontab -
-  done
-fi
-
-echo "Starting cron in foreground."
-crond -f -d 8

go.mod

@@ -10,12 +10,14 @@ require (
 	github.com/docker/cli v24.0.1+incompatible
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/gofrs/flock v0.8.1
-	github.com/klauspost/compress v1.17.4
+	github.com/joho/godotenv v1.5.1
+	github.com/klauspost/compress v1.17.6
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/offen/envconfig v1.5.0
 	github.com/otiai10/copy v1.14.0
 	github.com/pkg/sftp v1.13.6
+	github.com/robfig/cron/v3 v3.0.0
 	github.com/studio-b12/gowebdav v0.9.0
 	golang.org/x/crypto v0.18.0
 	golang.org/x/oauth2 v0.16.0

go.sum

@@ -443,6 +443,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc=
 github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -456,8 +458,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@@ -593,6 +595,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
+github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=

test/confd/run.sh

@@ -13,6 +13,8 @@ docker compose up -d --quiet-pull
 # sleep until a backup is guaranteed to have happened on the 1 minute schedule
 sleep 100
 
+docker compose logs backup
+
 if [ ! -f "$LOCAL_DIR/conf.tar.gz" ]; then
   fail "Config from file was not used."
 fi

test/lock/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    environment:
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: '7'
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${LOCAL_DIR:-./local}:/archive
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/lock/run.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+
+ec=0
+
+docker compose exec -e BACKUP_RETENTION_DAYS=7 -e BACKUP_FILENAME=test.tar.gz backup backup & \
+  { set +e; sleep 0.1; docker compose exec -e BACKUP_FILENAME=test2.tar.gz -e LOCK_TIMEOUT=1s backup backup; ec=$?;}
+
+if [ "$ec" = "0" ]; then
+  fail "Subsequent invocation exited 0"
+fi
+pass "Subsequent invocation did not exit 0"
+
+sleep 5
+
+if [ ! -f "${LOCAL_DIR}/test.tar.gz" ]; then
+  fail "Could not find expected tar file"
+fi
+pass "Found expected tar file"
+
+if [ -f "${LOCAL_DIR}/test2.tar.gz" ]; then
+  fail "Subsequent invocation was expected to fail but created archive"
+fi
+pass "Subsequent invocation did not create archive"

test/proxy/docker-compose.swarm.yml

@@ -0,0 +1,40 @@
+# Copyright 2020-2021 - Offen Authors <hioffen@posteo.de>
+# SPDX-License-Identifier: Unlicense
+
+version: '3.8'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      DOCKER_HOST: tcp://docker_socket_proxy:2375
+    volumes:
+      - pg_data:/backup/pg_data:ro
+      - ${LOCAL_DIR:-local}:/archive
+
+  docker_socket_proxy:
+    image: tecnativa/docker-socket-proxy:0.1
+    environment:
+      INFO: ${ALLOW_INFO:-1}
+      CONTAINERS: ${ALLOW_CONTAINERS:-1}
+      SERVICES: ${ALLOW_SERVICES:-1}
+      POST: ${ALLOW_POST:-1}
+      TASKS: ${ALLOW_TASKS:-1}
+      NODES: ${ALLOW_NODES:-1}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  pg:
+    image: postgres:14-alpine
+    environment:
+      POSTGRES_PASSWORD: example
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    deploy:
+      labels:
+        - docker-volume-backup.stop-during-backup=true
+
+volumes:
+  pg_data:

test/proxy/docker-compose.yml

@@ -0,0 +1,36 @@
+# Copyright 2020-2021 - Offen Authors <hioffen@posteo.de>
+# SPDX-License-Identifier: Unlicense
+
+version: '3.8'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      DOCKER_HOST: tcp://docker_socket_proxy:2375
+    volumes:
+      - pg_data:/backup/pg_data:ro
+      - ${LOCAL_DIR:-local}:/archive
+
+  docker_socket_proxy:
+    image: tecnativa/docker-socket-proxy:0.1
+    environment:
+      INFO: ${ALLOW_INFO:-1}
+      CONTAINERS: ${ALLOW_CONTAINERS:-1}
+      POST: ${ALLOW_POST:-1}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  pg:
+    image: postgres:14-alpine
+    environment:
+      POSTGRES_PASSWORD: example
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+
+volumes:
+  pg_data:

test/proxy/run.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+
+# The default configuration in docker-compose.yml should
+# successfully create a backup.
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then
+  fail "Archive was not created"
+fi
+pass "Found relevant archive file."
+
+# Disabling POST should make the backup run fail
+ALLOW_POST="0" docker compose up -d
+sleep 5
+
+set +e
+docker compose exec backup backup
+if [ $? = "0" ]; then
+  fail "Expected invocation to exit non-zero."
+fi
+set -e
+pass "Invocation exited non-zero."
+
+docker compose down --volumes
+
+# Next, the test is run against a Swarm setup
+
+docker swarm init
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker stack deploy --compose-file=docker-compose.swarm.yml test_stack
+
+sleep 20
+
+# The default configuration in docker-compose.swarm.yml should
+# successfully create a backup in Swarm mode.
+docker exec $(docker ps -q -f name=backup) backup
+
+if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then
+  fail "Archive was not created"
+fi
+
+pass "Found relevant archive file."
+
+sleep 5
+expect_running_containers "3"
+
+# Disabling POST should make the backup run fail
+ALLOW_POST="0" docker stack deploy --compose-file=docker-compose.swarm.yml test_stack
+
+sleep 20
+
+set +e
+docker exec $(docker ps -q -f name=backup) backup
+if [ $? = "0" ]; then
+  fail "Expected invocation to exit non-zero."
+fi
+set -e
+
+pass "Invocation exited non-zero."