Compare commits

...

4 Commits

Author SHA1 Message Date
Frederik Ring
6034e6a902 print proper local archive in log message 2021-08-29 08:36:45 +02:00
Frederik Ring
d0eca0a179 fix container stop execution order 2021-08-26 16:22:24 +02:00
Frederik Ring
a0fe2cf42d handle errors on container restart 2021-08-26 12:50:22 +02:00
Frederik Ring
5334ff1a5a refactor script initialization 2021-08-25 07:48:20 +02:00
2 changed files with 72 additions and 59 deletions

View File

@@ -2,7 +2,7 @@
Backup Docker volumes locally or to any S3 compatible storage. Backup Docker volumes locally or to any S3 compatible storage.
The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup. It handles recurring backups of Docker volumes to a local directory or any S3 compatible storage (or both), and rotates away old backups if configured. The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup. It handles __recurring or one-off backups of Docker volumes__ to a __local directory__ or __any S3 compatible storage__ (or both), and __rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__.
## Configuration ## Configuration

View File

@@ -19,7 +19,7 @@ import (
"github.com/gofrs/flock" "github.com/gofrs/flock"
"github.com/kelseyhightower/envconfig" "github.com/kelseyhightower/envconfig"
"github.com/leekchan/timeutil" "github.com/leekchan/timeutil"
minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/credentials"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/walle/targz" "github.com/walle/targz"
@@ -30,12 +30,25 @@ func main() {
unlock := lock("/var/lock/dockervolumebackup.lock") unlock := lock("/var/lock/dockervolumebackup.lock")
defer unlock() defer unlock()
s := &script{} s, err := newScript()
s.must(s.init()) if err != nil {
s.must(s.stopContainersAndRun(s.takeBackup)) panic(err)
}
s.must(func() error {
restartContainers, err := s.stopContainers()
defer func() {
s.must(restartContainers())
}()
if err != nil {
return err
}
return s.takeBackup()
}())
s.must(s.encryptBackup()) s.must(s.encryptBackup())
s.must(s.copyBackup()) s.must(s.copyBackup())
s.must(s.cleanBackup()) s.must(s.removeArtifacts())
s.must(s.pruneOldBackups()) s.must(s.pruneOldBackups())
s.logger.Info("Finished running backup tasks.") s.logger.Info("Finished running backup tasks.")
} }
@@ -71,25 +84,34 @@ type config struct {
GpgPassphrase string `split_words:"true"` GpgPassphrase string `split_words:"true"`
} }
// init creates all resources needed for the script to perform actions against // newScript creates all resources needed for the script to perform actions against
// remote resources like the Docker engine or remote storage locations. All // remote resources like the Docker engine or remote storage locations. All
// reading from env vars or other configuration sources is expected to happen // reading from env vars or other configuration sources is expected to happen
// in this method. // in this method.
func (s *script) init() error { func newScript() (*script, error) {
s.ctx = context.Background() s := &script{
s.logger = logrus.New() c: &config{},
s.logger.SetOutput(os.Stdout) ctx: context.Background(),
logger: &logrus.Logger{
s.c = &config{} Out: os.Stdout,
if err := envconfig.Process("", s.c); err != nil { Formatter: new(logrus.TextFormatter),
return fmt.Errorf("init: failed to process configuration values: %w", err) Hooks: make(logrus.LevelHooks),
Level: logrus.InfoLevel,
},
start: time.Now(),
} }
if err := envconfig.Process("", s.c); err != nil {
return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
}
s.file = path.Join("/tmp", s.c.BackupFilename)
_, err := os.Stat("/var/run/docker.sock") _, err := os.Stat("/var/run/docker.sock")
if !os.IsNotExist(err) { if !os.IsNotExist(err) {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil { if err != nil {
return fmt.Errorf("init: failed to create docker client") return nil, fmt.Errorf("newScript: failed to create docker client")
} }
s.cli = cli s.cli = cli
} }
@@ -104,31 +126,29 @@ func (s *script) init() error {
Secure: !s.c.AwsEndpointInsecure && s.c.AwsEndpointProto == "https", Secure: !s.c.AwsEndpointInsecure && s.c.AwsEndpointProto == "https",
}) })
if err != nil { if err != nil {
return fmt.Errorf("init: error setting up minio client: %w", err) return nil, fmt.Errorf("newScript: error setting up minio client: %w", err)
} }
s.mc = mc s.mc = mc
} }
s.file = path.Join("/tmp", s.c.BackupFilename) return s, nil
s.start = time.Now()
return nil
} }
// stopContainersAndRun stops all Docker containers that are marked as to being var noop = func() error { return nil }
// stopped during the backup and runs the given thunk. After returning, it makes
// sure containers are being restarted if required. In case the docker cli // stopContainers stops all Docker containers that are marked as to being
// is not configured, it will call the given thunk immediately. // stopped during the backup and returns a function that can be called to
func (s *script) stopContainersAndRun(thunk func() error) error { // restart everything that has been stopped.
func (s *script) stopContainers() (func() error, error) {
if s.cli == nil { if s.cli == nil {
return thunk() return noop, nil
} }
allContainers, err := s.cli.ContainerList(s.ctx, types.ContainerListOptions{ allContainers, err := s.cli.ContainerList(s.ctx, types.ContainerListOptions{
Quiet: true, Quiet: true,
}) })
if err != nil { if err != nil {
return fmt.Errorf("stopContainersAndRun: error querying for containers: %w", err) return noop, fmt.Errorf("stopContainersAndRun: error querying for containers: %w", err)
} }
containerLabel := fmt.Sprintf( containerLabel := fmt.Sprintf(
@@ -144,11 +164,11 @@ func (s *script) stopContainersAndRun(thunk func() error) error {
}) })
if err != nil { if err != nil {
return fmt.Errorf("stopContainersAndRun: error querying for containers to stop: %w", err) return noop, fmt.Errorf("stopContainersAndRun: error querying for containers to stop: %w", err)
} }
if len(containersToStop) == 0 { if len(containersToStop) == 0 {
return thunk() return noop, nil
} }
s.logger.Infof( s.logger.Infof(
@@ -168,7 +188,15 @@ func (s *script) stopContainersAndRun(thunk func() error) error {
} }
} }
defer func() error { if len(stopErrors) != 0 {
return noop, fmt.Errorf(
"stopContainersAndRun: %d error(s) stopping containers: %w",
len(stopErrors),
err,
)
}
return func() error {
servicesRequiringUpdate := map[string]struct{}{} servicesRequiringUpdate := map[string]struct{}{}
var restartErrors []error var restartErrors []error
@@ -193,7 +221,7 @@ func (s *script) stopContainersAndRun(thunk func() error) error {
} }
} }
if serviceMatch.ID == "" { if serviceMatch.ID == "" {
return fmt.Errorf("stopContainersAndRun: Couldn't find service with name %s", serviceName) return fmt.Errorf("stopContainersAndRun: couldn't find service with name %s", serviceName)
} }
serviceMatch.Spec.TaskTemplate.ForceUpdate = 1 serviceMatch.Spec.TaskTemplate.ForceUpdate = 1
_, err := s.cli.ServiceUpdate( _, err := s.cli.ServiceUpdate(
@@ -218,17 +246,7 @@ func (s *script) stopContainersAndRun(thunk func() error) error {
len(stoppedContainers), len(stoppedContainers),
) )
return nil return nil
}() }, nil
if len(stopErrors) != 0 {
return fmt.Errorf(
"stopContainersAndRun: %d error(s) stopping containers: %w",
len(stopErrors),
err,
)
}
return thunk()
} }
// takeBackup creates a tar archive of the configured backup location and // takeBackup creates a tar archive of the configured backup location and
@@ -249,6 +267,7 @@ func (s *script) encryptBackup() error {
if s.c.GpgPassphrase == "" { if s.c.GpgPassphrase == "" {
return nil return nil
} }
defer os.Remove(s.file)
gpgFile := fmt.Sprintf("%s.gpg", s.file) gpgFile := fmt.Sprintf("%s.gpg", s.file)
outFile, err := os.Create(gpgFile) outFile, err := os.Create(gpgFile)
@@ -276,10 +295,6 @@ func (s *script) encryptBackup() error {
return fmt.Errorf("encryptBackup: error writing ciphertext to file: %w", err) return fmt.Errorf("encryptBackup: error writing ciphertext to file: %w", err)
} }
if err := os.Remove(s.file); err != nil {
return fmt.Errorf("encryptBackup: error removing unencrpyted backup: %w", err)
}
s.file = gpgFile s.file = gpgFile
s.logger.Infof("Encrypted backup using given passphrase, saving as `%s`.", s.file) s.logger.Infof("Encrypted backup using given passphrase, saving as `%s`.", s.file)
return nil return nil
@@ -303,17 +318,17 @@ func (s *script) copyBackup() error {
if err := copy(s.file, path.Join(s.c.BackupArchive, name)); err != nil { if err := copy(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
return fmt.Errorf("copyBackup: error copying file to local archive: %w", err) return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
} }
s.logger.Infof("Stored copy of backup `%s` in local archive `%s`", s.file, s.c.AwsS3BucketName) s.logger.Infof("Stored copy of backup `%s` in local archive `%s`", s.file, s.c.BackupArchive)
} }
return nil return nil
} }
// cleanBackup removes the backup file from disk. // removeArtifacts removes the backup file from disk.
func (s *script) cleanBackup() error { func (s *script) removeArtifacts() error {
if err := os.Remove(s.file); err != nil { if err := os.Remove(s.file); err != nil {
return fmt.Errorf("cleanBackup: error removing file: %w", err) return fmt.Errorf("removeArtifacts: error removing file: %w", err)
} }
s.logger.Info("Cleaned up local artifacts.") s.logger.Info("Removed local artifacts.")
return nil return nil
} }
@@ -378,9 +393,10 @@ func (s *script) pruneOldBackups() error {
) )
} }
s.logger.Infof( s.logger.Infof(
"Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period.", "Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.",
len(matches), len(matches),
lenCandidates, lenCandidates,
s.c.BackupRetentionDays,
) )
} else if len(matches) != 0 && len(matches) == lenCandidates { } else if len(matches) != 0 && len(matches) == lenCandidates {
s.logger.Warnf( s.logger.Warnf(
@@ -434,9 +450,10 @@ func (s *script) pruneOldBackups() error {
) )
} }
s.logger.Infof( s.logger.Infof(
"Pruned %d out of %d local backup(s) as their age exceeded the configured retention period.", "Pruned %d out of %d local backup(s) as their age exceeded the configured retention period of %d days.",
len(matches), len(matches),
len(candidates), len(candidates),
s.c.BackupRetentionDays,
) )
} else if len(matches) != 0 && len(matches) == len(candidates) { } else if len(matches) != 0 && len(matches) == len(candidates) {
s.logger.Warnf( s.logger.Warnf(
@@ -453,11 +470,7 @@ func (s *script) pruneOldBackups() error {
func (s *script) must(err error) { func (s *script) must(err error) {
if err != nil { if err != nil {
if s.logger == nil { s.logger.Fatalf("Fatal error running backup: %s", err)
panic(err)
}
s.logger.Errorf("Fatal error running backup: %s", err)
os.Exit(1)
} }
} }