Improve Swarm support (#333)

* Query for labeled services as well

* Try scaling down services

* Scale services back up

* Use progress tool from Docker CLI

* In test, label both services

* Clean up error and log messages

* Document scale-up/down approach in docs

* Downgrade Docker CLI to match client

* Document services stats

* Do not rely on PreviousSpec for storing desired replica count

* Log warnings from Docker when updating services

* Check whether container and service labels collide

* Document script behavior on label collision

* Add additional check if all containers have been removed

* Scale services concurrently

* Move docker interaction code into own file

* Factor out code for service updating

* Time out after five minutes of not reaching desired container count

* Inline handling of in-swarm container level restart

* Timer is more suitable for timeout race

* Timeout when scaling down services should be configurable

* Choose better filename

* Reflect changes in naming

* Rename and deprecate BACKUP_STOP_CONTAINER_LABEL

* Improve logging

* Further simplify logging

cmd/backup/stop_restart.go

@@ -170,26 +170,23 @@ func (s *script) stopContainersAndServices() (func() error, error) {
 		}
 	}
 
+	s.logger.Info(
+		fmt.Sprintf(
+			"Stopping %d out of %d running container(s) as they were labeled %s.",
+			len(containersToStop),
+			len(allContainers),
+			filterMatchLabel,
+		),
+	)
 	if isDockerSwarm {
 		s.logger.Info(
 			fmt.Sprintf(
-				"Stopping %d out of %d running container(s) and scaling down %d out of %d active service(s) as they were labeled %s.",
-				len(containersToStop),
-				len(allContainers),
+				"Scaling down %d out of %d active service(s) as they were labeled %s.",
 				len(servicesToScaleDown),
 				len(allServices),
 				filterMatchLabel,
 			),
 		)
-	} else {
-		s.logger.Info(
-			fmt.Sprintf(
-				"Stopping %d out of %d running container(s) as they were labeled %s.",
-				len(containersToStop),
-				len(allContainers),
-				filterMatchLabel,
-			),
-		)
 	}
 
 	var stoppedContainers []types.Container
@@ -320,21 +317,20 @@ func (s *script) stopContainersAndServices() (func() error, error) {
 				errors.Join(allErrors...),
 			)
 		}
+
+		s.logger.Info(
+			fmt.Sprintf(
+				"Restarted %d container(s).",
+				len(stoppedContainers),
+			),
+		)
 		if isDockerSwarm {
 			s.logger.Info(
 				fmt.Sprintf(
-					"Restarted %d container(s) and %d service(s).",
-					len(stoppedContainers),
+					"Scaled %d service(s) back up.",
 					len(scaledDownServices),
 				),
 			)
-		} else {
-			s.logger.Info(
-				fmt.Sprintf(
-					"Restarted %d container(s).",
-					len(stoppedContainers),
-				),
-			)
 		}
 
 		return nil

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/docker/cli v24.0.1+incompatible
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/gofrs/flock v0.8.1
-	github.com/klauspost/compress v1.17.4
+	github.com/klauspost/compress v1.17.5
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/offen/envconfig v1.5.0

go.sum

@@ -456,8 +456,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E=
+github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=