Run httpd server in container if requested

* Current Docker client breaks in newer Go versions

* Cater for breaking API changes in Docker client

* Update Docker client

* Unpin Go version used for build

* Tidy sum file

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly

.github/workflows/test.yml

@@ -1,18 +1,30 @@
 name: Run Integration Tests
 
 on:
-  - push
-  - pull_request
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
   test:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Build Docker Image
+        env:
+          DOCKER_BUILDKIT: '1'
         run: docker build . -t offen/docker-volume-backup:test
+
       - name: Run Tests
         working-directory: ./test
         run: |
+          # Stop the buildx container so the tests can make assertions
+          # about the number of running containers
+          docker rm -f $(docker ps -aq)
           export GPG_TTY=$(tty)
           ./test.sh test

Dockerfile

@@ -9,15 +9,13 @@ RUN go mod download
 WORKDIR /app/cmd/backup
 RUN go build -o backup .
 
-FROM alpine:3.17
+FROM alpine:3.18
 
 WORKDIR /root
 
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates busybox-extras
 
 COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
-
-COPY ./entrypoint.sh /root/
-RUN chmod +x entrypoint.sh
+COPY --chmod=755 ./entrypoint.sh /root/
 
 ENTRYPOINT ["/root/entrypoint.sh"]

README.md

@@ -14,6 +14,7 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
 - [Quickstart](#quickstart)
   - [Recurring backups in a compose setup](#recurring-backups-in-a-compose-setup)
   - [One-off backups using Docker CLI](#one-off-backups-using-docker-cli)
+  - [Available image registries](#available-image-registries)
 - [Configuration reference](#configuration-reference)
 - [How to](#how-to)
   - [Stop containers during backup](#stop-containers-during-backup)
@@ -121,6 +122,18 @@ docker run --rm \
 
 Alternatively, pass a `--env-file` in order to use a full config as described below.
 
+### Available image registries
+
+This Docker image is published to both Docker Hub and the GitHub container registry.
+Depending on your preferences and needs, you can reference both `offen/docker-volume-backup` as well as `ghcr.io/offen/docker-volume-backup`:
+
+```
+docker pull offen/docker-volume-backup:v2
+docker pull ghcr.io/offen/docker-volume-backup:v2
+```
+
+Documentation references Docker Hub, but all examples will work using ghcr.io just as well.
+
 ## Configuration reference
 
 Backup targets, schedule and retention are configured in environment variables.
@@ -247,6 +260,15 @@ You can populate below template according to your requirements and use it as you
 
 # AWS_STORAGE_CLASS="GLACIER"
 
+# Setting this variable will change the S3 default part size for the copy step.
+# This value is useful when you want to upload large files.
+# NB : While using Scaleway as S3 provider, be aware that the parts counter is set to 1.000.
+# While Minio uses a hard coded value to 10.000. As a workaround, try to set a higher value.
+# Defaults to "16" (MB) if unset (from minio), you can set this value according to your needs.
+# The unit is in MB and an integer.
+
+# AWS_PART_SIZE=16
+
 # You can also backup files to any WebDAV server:
 
 # The URL of the remote WebDAV server
@@ -411,7 +433,7 @@ You can populate below template according to your requirements and use it as you
 
 # Notifications (email, Slack, etc.) can be sent out when a backup run finishes.
 # Configuration is provided as a comma-separated list of URLs as consumed
-# by `shoutrrr`: https://containrrr.dev/shoutrrr/v0.5/services/overview/
+# by `shoutrrr`: https://containrrr.dev/shoutrrr/0.7/services/overview/
 # The content of such notifications can be customized. Dedicated documentation
 # on how to do this can be found in the README. When providing multiple URLs or
 # an URL that contains a comma, the values can be URL encoded to avoid ambiguities.
@@ -555,7 +577,7 @@ services:
 Notification backends other than email are also supported.
 Refer to the documentation of [shoutrrr][shoutrrr-docs] to find out about options and configuration.
 
-[shoutrrr-docs]: https://containrrr.dev/shoutrrr/v0.5/services/overview/
+[shoutrrr-docs]: https://containrrr.dev/shoutrrr/0.7/services/overview/
 
 ### Customize notifications
 
@@ -644,7 +666,23 @@ volumes:
 The backup procedure is guaranteed to wait for all `pre` or `post` commands to finish before proceeding.
 However there are no guarantees about the order in which they are run, which could also happen concurrently.
 
-By default the backup command is executed by the root user. It is possible to specify a custom user in container labels with the format `docker-volume-backup.[step]-[pre|post]-[user]`. The option will allow you to run a specific step command by specified user. Make sure the user exists and present in `passwd` inside the target container.
+By default the backup command is executed by the user provided by the container's image.
+It is possible to specify a custom user that is used to run commands in dedicated labels with the format `docker-volume-backup.[step]-[pre|post].user`:
+
+```yml
+version: '3'
+
+services:
+  gitea:
+    image: gitea/gitea
+    volumes:
+      - backup_data:/tmp
+    labels:
+      - docker-volume-backup.archive-pre.user=git
+      - docker-volume-backup.archive-pre=/bin/bash -c 'cd /tmp; /usr/local/bin/gitea dump -c /data/gitea/conf/app.ini -R -f dump.zip'
+```
+
+Make sure the user exists and is present in `passwd` inside the target container.
 
 ### Encrypting your backup using GPG
 
@@ -890,7 +928,7 @@ BACKUP_SOURCES=/backup/app2_data
 
 If you want to manage backup retention on different schedules, the most straight forward approach is to define a dedicated configuration for retention rule using a different prefix in the `BACKUP_FILENAME` parameter and then run them on different cron schedules.
 
-For example, if you wanted to keep daily backups for 7 days, weekly backups for a month, and retain monthly backups forever, you could create three configuration files and mount them into `/etc/dockervolumebackup.d`:
+For example, if you wanted to keep daily backups for 7 days, weekly backups for a month, and retain monthly backups forever, you could create three configuration files and mount them into `/etc/dockervolumebackup/conf.d`:
 
 ```ini
 # 01daily.conf

cmd/backup/config.go

@@ -28,6 +28,7 @@ type Config struct {
 	AwsSecretAccessKey            string        `split_words:"true"`
 	AwsSecretAccessKeyFile        string        `split_words:"true"`
 	AwsIamRoleEndpoint            string        `split_words:"true"`
+	AwsPartSize                   int64         `split_words:"true"`
 	BackupSources                 string        `split_words:"true" default:"/backup"`
 	BackupFilename                string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
 	BackupFilenameExpand          bool          `split_words:"true"`

cmd/backup/exec.go

@@ -91,7 +91,6 @@ func (s *script) runLabeledCommands(label string) error {
 		})
 	}
 	containersWithCommand, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Quiet:   true,
 		Filters: filters.NewArgs(f...),
 	})
 	if err != nil {
@@ -105,7 +104,6 @@ func (s *script) runLabeledCommands(label string) error {
 			Value: "docker-volume-backup.exec-pre",
 		}
 		deprecatedContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-			Quiet:   true,
 			Filters: filters.NewArgs(f...),
 		})
 		if err != nil {
@@ -123,7 +121,6 @@ func (s *script) runLabeledCommands(label string) error {
 			Value: "docker-volume-backup.exec-post",
 		}
 		deprecatedContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-			Quiet:   true,
 			Filters: filters.NewArgs(f...),
 		})
 		if err != nil {

cmd/backup/script.go

@@ -25,6 +25,7 @@ import (
 	"github.com/containrrr/shoutrrr"
 	"github.com/containrrr/shoutrrr/pkg/router"
 	"github.com/docker/docker/api/types"
+	ctr "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/client"
@@ -142,6 +143,7 @@ func newScript() (*script, error) {
 			BucketName:       s.c.AwsS3BucketName,
 			StorageClass:     s.c.AwsStorageClass,
 			CACert:           s.c.AwsEndpointCACert.Cert,
+			PartSize:         s.c.AwsPartSize,
 		}
 		if s3Backend, err := s3.NewStorageBackend(s3Config, logFunc); err != nil {
 			return nil, err
@@ -280,9 +282,7 @@ func (s *script) stopContainers() (func() error, error) {
 		return noop, nil
 	}
 
-	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Quiet: true,
-	})
+	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{})
 	if err != nil {
 		return noop, fmt.Errorf("stopContainers: error querying for containers: %w", err)
 	}
@@ -292,7 +292,6 @@ func (s *script) stopContainers() (func() error, error) {
 		s.c.BackupStopContainerLabel,
 	)
 	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
-		Quiet: true,
 		Filters: filters.NewArgs(filters.KeyValuePair{
 			Key:   "label",
 			Value: containerLabel,
@@ -317,7 +316,7 @@ func (s *script) stopContainers() (func() error, error) {
 	var stoppedContainers []types.Container
 	var stopErrors []error
 	for _, container := range containersToStop {
-		if err := s.cli.ContainerStop(context.Background(), container.ID, nil); err != nil {
+		if err := s.cli.ContainerStop(context.Background(), container.ID, ctr.StopOptions{}); err != nil {
 			stopErrors = append(stopErrors, err)
 		} else {
 			stoppedContainers = append(stoppedContainers, container)
@@ -366,7 +365,7 @@ func (s *script) stopContainers() (func() error, error) {
 				if serviceMatch.ID == "" {
 					return fmt.Errorf("stopContainers: couldn't find service with name %s", serviceName)
 				}
-				serviceMatch.Spec.TaskTemplate.ForceUpdate = 1
+				serviceMatch.Spec.TaskTemplate.ForceUpdate += 1
 				if _, err := s.cli.ServiceUpdate(
 					context.Background(), serviceMatch.ID,
 					serviceMatch.Version, serviceMatch.Spec, types.ServiceUpdateOptions{},

entrypoint.sh

@@ -22,5 +22,12 @@ else
   done
 fi
 
+if [ ! -z "$SERVE_METRICS_PATH" ]; then
+  mkdir -p /var/www/html${SERVE_METRICS_PATH}
+  echo "ok" > /var/www/html${SERVE_METRICS_PATH}/metrics.txt
+  httpd -h /var/www/html -p "${SERVE_METRICS_PORT:-80}"
+  echo "Serving metrics on port ${SERVE_METRICS_PORT:-80}."
+fi
+
 echo "Starting cron in foreground."
 crond -f -d 8

go.mod

@@ -3,66 +3,59 @@ module github.com/offen/docker-volume-backup
 go 1.19
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1
-	github.com/containrrr/shoutrrr v0.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
+	github.com/containrrr/shoutrrr v0.7.1
 	github.com/cosiner/argv v0.1.0
-	github.com/docker/docker v20.10.11+incompatible
+	github.com/docker/docker v24.0.5+incompatible
 	github.com/gofrs/flock v0.8.1
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
-	github.com/minio/minio-go/v7 v7.0.44
-	github.com/otiai10/copy v1.7.0
+	github.com/minio/minio-go/v7 v7.0.61
+	github.com/otiai10/copy v1.11.0
 	github.com/pkg/sftp v1.13.5
-	github.com/sirupsen/logrus v1.9.0
-	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
-	golang.org/x/crypto v0.3.0
-	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
+	github.com/sirupsen/logrus v1.9.3
+	github.com/studio-b12/gowebdav v0.9.0
+	golang.org/x/crypto v0.11.0
+	golang.org/x/sync v0.3.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/containerd/containerd v1.6.18 // indirect
-	github.com/docker/distribution v2.8.0+incompatible // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/fatih/color v1.10.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/fatih/color v1.13.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.12 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.1 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kr/fs v0.1.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/moby/term v0.0.0-20200312100748-672ec06f55cd // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	github.com/onsi/gomega v1.10.3 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
-	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rs/xid v1.4.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect
-	google.golang.org/grpc v1.47.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gotest.tools/v3 v3.0.3 // indirect
 )

internal/storage/s3/s3.go

@@ -8,6 +8,7 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"os"
 	"path"
 	"path/filepath"
 	"time"
@@ -22,6 +23,7 @@ type s3Storage struct {
 	client       *minio.Client
 	bucket       string
 	storageClass string
+	partSize     int64
 }
 
 // Config contains values that define the configuration of a S3 backend.
@@ -35,6 +37,7 @@ type Config struct {
 	RemotePath       string
 	BucketName       string
 	StorageClass     string
+	PartSize         int64
 	CACert           *x509.Certificate
 }
 
@@ -89,6 +92,7 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error
 		client:       mc,
 		bucket:       opts.BucketName,
 		storageClass: opts.StorageClass,
+		partSize:     opts.PartSize,
 	}, nil
 }
 
@@ -100,16 +104,32 @@ func (v *s3Storage) Name() string {
 // Copy copies the given file to the S3/Minio storage backend.
 func (b *s3Storage) Copy(file string) error {
 	_, name := path.Split(file)
-
-	if _, err := b.client.FPutObject(context.Background(), b.bucket, filepath.Join(b.DestinationPath, name), file, minio.PutObjectOptions{
+	putObjectOptions := minio.PutObjectOptions{
 		ContentType:  "application/tar+gzip",
 		StorageClass: b.storageClass,
-	}); err != nil {
+	}
+
+	if b.partSize > 0 {
+		srcFileInfo, err := os.Stat(file)
+		if err != nil {
+			return fmt.Errorf("(*s3Storage).Copy: error reading the local file: %w", err)
+		}
+
+		_, partSize, _, err := minio.OptimalPartInfo(srcFileInfo.Size(), uint64(b.partSize*1024*1024))
+		if err != nil {
+			return fmt.Errorf("(*s3Storage).Copy: error computing the optimal s3 part size: %w", err)
+		}
+
+		putObjectOptions.PartSize = uint64(partSize)
+	}
+
+	if _, err := b.client.FPutObject(context.Background(), b.bucket, filepath.Join(b.DestinationPath, name), file, putObjectOptions); err != nil {
 		if errResp := minio.ToErrorResponse(err); errResp.Message != "" {
 			return fmt.Errorf("(*s3Storage).Copy: error uploading backup to remote storage: [Message]: '%s', [Code]: %s, [StatusCode]: %d", errResp.Message, errResp.Code, errResp.StatusCode)
 		}
 		return fmt.Errorf("(*s3Storage).Copy: error uploading backup to remote storage: %w", err)
 	}
+
 	b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to bucket `%s`.", file, b.bucket)
 
 	return nil

test/azure/run.sh

@@ -17,7 +17,7 @@ sleep 5
 
 expect_running_containers "3"
 
-docker-compose run --rm az_cli \
+docker compose run --rm az_cli \
   az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz
 tar -xvf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
 
@@ -26,7 +26,7 @@ pass "Found relevant files in untared remote backups."
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
 # TODO: find out if we can test actual deletion without having to wait for a day
-BACKUP_RETENTION_DAYS="0" docker-compose up -d
+BACKUP_RETENTION_DAYS="0" docker compose up -d
 sleep 5
 
 docker compose exec backup backup

test/commands/docker-compose.yml

@@ -42,10 +42,9 @@ services:
       EXEC_LABEL: test
       EXEC_FORWARD_OUTPUT: "true"
     volumes:
-      - archive:/archive
+      - ./local:/archive
       - app_data:/backup/data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 
 volumes:
   app_data:
-  archive:

test/commands/run.sh

@@ -6,11 +6,12 @@ cd $(dirname $0)
 . ../util.sh
 current_test=$(basename $(pwd))
 
+mkdir -p ./local
+
 docker compose up -d
 sleep 30 # mariadb likes to take a bit before responding
 
 docker compose exec backup backup
-sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' commands_archive) ./local
 
 tar -xvf ./local/test.tar.gz
 if [ ! -f ./backup/data/dump.sql ]; then
@@ -34,6 +35,7 @@ sudo rm -rf ./local
 
 info "Running commands test in swarm mode next."
 
+mkdir -p ./local
 docker swarm init
 
 docker stack deploy --compose-file=docker-compose.yml test_stack
@@ -47,8 +49,6 @@ sleep 20
 
 docker exec $(docker ps -q -f name=backup) backup
 
-sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' test_stack_archive) ./local
-
 tar -xvf ./local/test.tar.gz
 if [ ! -f ./backup/data/dump.sql ]; then
   fail "Could not find file written by pre command."