Compare commits

..

3 Commits

Author SHA1 Message Date
Frederik Ring
8b5c9a494f Exclude legacy docs page from navigation 2024-08-19 22:50:26 +02:00
nick comer
44ad3bbda2 feat: allow backups to be encrypted with age (#432)
GPG is known to have usability issues and is generally cumbersome to
use. age [0] is a modern alternative to GPG that is designed by a
cryptographer that has worked and continues to work on Golang's crypto
packages for years.

Allowing age to be used to encrypt backups dramatically simplifies the
backup process.

[0]: https://age-encryption.org/
2024-08-19 22:49:49 +02:00
dependabot[bot]
74e065cbb9 Bump github.com/minio/minio-go/v7 from 7.0.74 to 7.0.75 (#458) 2024-08-13 04:05:47 +00:00
15 changed files with 413 additions and 193 deletions

View File

@@ -36,9 +36,6 @@ func (c *command) runAsCommand() error {
} }
for _, config := range configurations { for _, config := range configurations {
if err := config.validate(); err != nil {
return errwrap.Wrap(err, "error validating config")
}
if err := runScript(config); err != nil { if err := runScript(config); err != nil {
return errwrap.Wrap(err, "error running script") return errwrap.Wrap(err, "error running script")
} }
@@ -104,12 +101,6 @@ func (c *command) schedule(strategy configStrategy) error {
} }
for _, cfg := range configurations { for _, cfg := range configurations {
if err := cfg.validate(); err != nil {
return errwrap.Wrap(
err,
fmt.Sprintf("error validating config for schedule %s", cfg.BackupCronExpression),
)
}
config := cfg config := cfg
id, err := c.cr.AddFunc(config.BackupCronExpression, func() { id, err := c.cr.AddFunc(config.BackupCronExpression, func() {
c.logger.Info( c.logger.Info(

View File

@@ -12,91 +12,85 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/offen/docker-volume-backup/internal/errwrap" "github.com/offen/docker-volume-backup/internal/errwrap"
) )
// Config holds all configuration values that are expected to be set // Config holds all configuration values that are expected to be set
// by users. // by users.
type Config struct { type Config struct {
AwsS3BucketName string `split_words:"true"` AwsS3BucketName string `split_words:"true"`
AwsS3Path string `split_words:"true"` AwsS3Path string `split_words:"true"`
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
AwsEndpointProto string `split_words:"true" default:"https"` AwsEndpointProto string `split_words:"true" default:"https"`
AwsEndpointInsecure bool `split_words:"true"` AwsEndpointInsecure bool `split_words:"true"`
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
AwsStorageClass string `split_words:"true"` AwsStorageClass string `split_words:"true"`
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsSecretAccessKey string `split_words:"true"` AwsSecretAccessKey string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"` AwsIamRoleEndpoint string `split_words:"true"`
AwsPartSize int64 `split_words:"true"` AwsPartSize int64 `split_words:"true"`
BackupCompression CompressionType `split_words:"true" default:"gz"` BackupCompression CompressionType `split_words:"true" default:"gz"`
GzipParallelism WholeNumber `split_words:"true" default:"1"` GzipParallelism WholeNumber `split_words:"true" default:"1"`
BackupSources string `split_words:"true" default:"/backup"` BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"` BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"`
BackupFilenameExpand bool `split_words:"true"` BackupFilenameExpand bool `split_words:"true"`
BackupLatestSymlink string `split_words:"true"` BackupLatestSymlink string `split_words:"true"`
BackupArchive string `split_words:"true" default:"/archive"` BackupArchive string `split_words:"true" default:"/archive"`
BackupCronExpression string `split_words:"true" default:"@daily"` BackupCronExpression string `split_words:"true" default:"@daily"`
BackupRetentionDays int32 `split_words:"true" default:"-1"` BackupRetentionDays int32 `split_words:"true" default:"-1"`
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
BackupPruningPrefix string `split_words:"true"` BackupPruningPrefix string `split_words:"true"`
BackupStopContainerLabel string `split_words:"true"` BackupStopContainerLabel string `split_words:"true"`
BackupStopDuringBackupLabel string `split_words:"true" default:"true"` BackupStopDuringBackupLabel string `split_words:"true" default:"true"`
BackupStopServiceTimeout time.Duration `split_words:"true" default:"5m"` BackupStopServiceTimeout time.Duration `split_words:"true" default:"5m"`
BackupFromSnapshot bool `split_words:"true"` BackupFromSnapshot bool `split_words:"true"`
BackupExcludeRegexp RegexpDecoder `split_words:"true"` BackupExcludeRegexp RegexpDecoder `split_words:"true"`
BackupSkipBackendsFromPrune []string `split_words:"true"` BackupSkipBackendsFromPrune []string `split_words:"true"`
GpgPassphrase string `split_words:"true"` GpgPassphrase string `split_words:"true"`
GpgPublicKeyRing string `split_words:"true"` GpgPublicKeyRing string `split_words:"true"`
NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` AgePassphrase string `split_words:"true"`
NotificationLevel string `split_words:"true" default:"error"` AgePublicKeys []string `split_words:"true"`
EmailNotificationRecipient string `split_words:"true"` NotificationURLs []string `envconfig:"NOTIFICATION_URLS"`
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` NotificationLevel string `split_words:"true" default:"error"`
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` EmailNotificationRecipient string `split_words:"true"`
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
WebdavUrl string `split_words:"true"` EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
WebdavUrlInsecure bool `split_words:"true"` EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
WebdavPath string `split_words:"true" default:"/"` WebdavUrl string `split_words:"true"`
WebdavUsername string `split_words:"true"` WebdavUrlInsecure bool `split_words:"true"`
WebdavPassword string `split_words:"true"` WebdavPath string `split_words:"true" default:"/"`
SSHHostName string `split_words:"true"` WebdavUsername string `split_words:"true"`
SSHPort string `split_words:"true" default:"22"` WebdavPassword string `split_words:"true"`
SSHUser string `split_words:"true"` SSHHostName string `split_words:"true"`
SSHPassword string `split_words:"true"` SSHPort string `split_words:"true" default:"22"`
SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"` SSHUser string `split_words:"true"`
SSHIdentityPassphrase string `split_words:"true"` SSHPassword string `split_words:"true"`
SSHRemotePath string `split_words:"true"` SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"`
ExecLabel string `split_words:"true"` SSHIdentityPassphrase string `split_words:"true"`
ExecForwardOutput bool `split_words:"true"` SSHRemotePath string `split_words:"true"`
LockTimeout time.Duration `split_words:"true" default:"60m"` ExecLabel string `split_words:"true"`
AzureStorageAccountName string `split_words:"true"` ExecForwardOutput bool `split_words:"true"`
AzureStoragePrimaryAccountKey string `split_words:"true"` LockTimeout time.Duration `split_words:"true" default:"60m"`
AzureStorageConnectionString string `split_words:"true"` AzureStorageAccountName string `split_words:"true"`
AzureStorageContainerName string `split_words:"true"` AzureStoragePrimaryAccountKey string `split_words:"true"`
AzureStoragePath string `split_words:"true"` AzureStorageConnectionString string `split_words:"true"`
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` AzureStorageContainerName string `split_words:"true"`
AzureStorageAccessTier AzureStorageAccessTier `split_words:"true"` AzureStoragePath string `split_words:"true"`
DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"` AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"` AzureStorageAccessTier string `split_words:"true"`
DropboxRefreshToken string `split_words:"true"` DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"`
DropboxAppKey string `split_words:"true"` DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"`
DropboxAppSecret string `split_words:"true"` DropboxRefreshToken string `split_words:"true"`
DropboxRemotePath string `split_words:"true"` DropboxAppKey string `split_words:"true"`
DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"` DropboxAppSecret string `split_words:"true"`
DropboxRemotePath string `split_words:"true"`
DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"`
source string source string
additionalEnvVars map[string]string additionalEnvVars map[string]string
} }
func (c *Config) validate() error {
if c.AzureStoragePrimaryAccountKey != "" && c.AzureStorageConnectionString != "" {
return errwrap.Wrap(nil, "using azure primary account key and connection string are mutually exclusive")
}
return nil
}
type CompressionType string type CompressionType string
func (c *CompressionType) Decode(v string) error { func (c *CompressionType) Decode(v string) error {
@@ -188,30 +182,6 @@ func (n *WholeNumber) Int() int {
return int(*n) return int(*n)
} }
type AzureStorageAccessTier string
func (t *AzureStorageAccessTier) Decode(v string) error {
if v == "" {
*t = ""
return nil
}
for _, a := range blob.PossibleAccessTierValues() {
if string(a) == v {
*t = AzureStorageAccessTier(v)
return nil
}
}
return errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", v))
}
func (t *AzureStorageAccessTier) AccessTier() *blob.AccessTier {
if *t == "" {
return nil
}
a := blob.AccessTier(*t)
return &a
}
type envVarLookup struct { type envVarLookup struct {
ok bool ok bool
key string key string

View File

@@ -11,109 +11,191 @@ import (
"os" "os"
"path" "path"
"filippo.io/age"
"github.com/ProtonMail/go-crypto/openpgp/armor" "github.com/ProtonMail/go-crypto/openpgp/armor"
openpgp "github.com/ProtonMail/go-crypto/openpgp/v2" openpgp "github.com/ProtonMail/go-crypto/openpgp/v2"
"github.com/offen/docker-volume-backup/internal/errwrap" "github.com/offen/docker-volume-backup/internal/errwrap"
) )
func (s *script) encryptAsymmetrically(outFile *os.File) (io.WriteCloser, func() error, error) { func countTrue(b ...bool) int {
c := int(0)
entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(s.c.GpgPublicKeyRing))) for _, v := range b {
if err != nil { if v {
return nil, nil, errwrap.Wrap(err, "error parsing armored keyring") c++
}
armoredWriter, err := armor.Encode(outFile, "PGP MESSAGE", nil)
if err != nil {
return nil, nil, errwrap.Wrap(err, "error preparing encryption")
}
_, name := path.Split(s.file)
dst, err := openpgp.Encrypt(armoredWriter, entityList, nil, nil, &openpgp.FileHints{
FileName: name,
}, nil)
if err != nil {
return nil, nil, err
}
return dst, func() error {
if err := dst.Close(); err != nil {
return err
} }
return armoredWriter.Close()
}, err
}
func (s *script) encryptSymmetrically(outFile *os.File) (io.WriteCloser, func() error, error) {
_, name := path.Split(s.file)
dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
FileName: name,
}, nil)
if err != nil {
return nil, nil, err
} }
return c
return dst, dst.Close, nil
} }
// encryptArchive encrypts the backup file using PGP and the configured passphrase or publickey(s). // encryptArchive encrypts the backup file using PGP and the configured passphrase or publickey(s).
// In case no passphrase or publickey is given it returns early, leaving the backup file // In case no passphrase or publickey is given it returns early, leaving the backup file
// untouched. // untouched.
func (s *script) encryptArchive() error { func (s *script) encryptArchive() error {
useGPGSymmetric := s.c.GpgPassphrase != ""
var encrypt func(outFile *os.File) (io.WriteCloser, func() error, error) useGPGAsymmetric := s.c.GpgPublicKeyRing != ""
var cleanUpErr error useAgeSymmetric := s.c.AgePassphrase != ""
useAgeAsymmetric := len(s.c.AgePublicKeys) > 0
switch { switch nconfigured := countTrue(
case s.c.GpgPassphrase != "" && s.c.GpgPublicKeyRing != "": useGPGSymmetric,
return errwrap.Wrap(nil, "error in selecting asymmetric and symmetric encryption methods: conflicting env vars are set") useGPGAsymmetric,
case s.c.GpgPassphrase != "": useAgeSymmetric,
encrypt = s.encryptSymmetrically useAgeAsymmetric,
case s.c.GpgPublicKeyRing != "": ); nconfigured {
encrypt = s.encryptAsymmetrically case 0:
default:
return nil return nil
case 1:
// ok!
default:
return fmt.Errorf(
"error in selecting archive encryption method: expected 0 or 1 to be configured, %d methods are configured",
nconfigured,
)
} }
gpgFile := fmt.Sprintf("%s.gpg", s.file) if useGPGSymmetric {
return s.encryptWithGPGSymmetric()
} else if useGPGAsymmetric {
return s.encryptWithGPGAsymmetric()
} else if useAgeSymmetric || useAgeAsymmetric {
ar, err := s.getConfiguredAgeRecipients()
if err != nil {
return errwrap.Wrap(err, "failed to get configured age recipients")
}
return s.encryptWithAge(ar)
}
return nil
}
func (s *script) getConfiguredAgeRecipients() ([]age.Recipient, error) {
if s.c.AgePassphrase == "" && len(s.c.AgePublicKeys) == 0 {
return nil, fmt.Errorf("no age recipients configured")
}
recipients := []age.Recipient{}
if len(s.c.AgePublicKeys) > 0 {
for _, pk := range s.c.AgePublicKeys {
pkr, err := age.ParseX25519Recipient(pk)
if err != nil {
return nil, errwrap.Wrap(err, "failed to parse age public key")
}
recipients = append(recipients, pkr)
}
}
if s.c.AgePassphrase != "" {
if len(recipients) != 0 {
return nil, fmt.Errorf("age encryption must only be enabled via passphrase or public key, not both")
}
r, err := age.NewScryptRecipient(s.c.AgePassphrase)
if err != nil {
return nil, errwrap.Wrap(err, "failed to create scrypt identity from age passphrase")
}
recipients = append(recipients, r)
}
return recipients, nil
}
func (s *script) encryptWithAge(rec []age.Recipient) error {
return s.doEncrypt("age", func(ciphertextWriter io.Writer) (io.WriteCloser, error) {
return age.Encrypt(ciphertextWriter, rec...)
})
}
func (s *script) encryptWithGPGSymmetric() error {
return s.doEncrypt("gpg", func(ciphertextWriter io.Writer) (io.WriteCloser, error) {
_, name := path.Split(s.file)
return openpgp.SymmetricallyEncrypt(ciphertextWriter, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
FileName: name,
}, nil)
})
}
type closeAllWriter struct {
io.Writer
closers []io.Closer
}
func (c *closeAllWriter) Close() (err error) {
for _, cl := range c.closers {
err = errors.Join(err, cl.Close())
}
return
}
var _ io.WriteCloser = (*closeAllWriter)(nil)
func (s *script) encryptWithGPGAsymmetric() error {
return s.doEncrypt("gpg", func(ciphertextWriter io.Writer) (_ io.WriteCloser, outerr error) {
entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(s.c.GpgPublicKeyRing)))
if err != nil {
return nil, errwrap.Wrap(err, "error parsing armored keyring")
}
armoredWriter, err := armor.Encode(ciphertextWriter, "PGP MESSAGE", nil)
if err != nil {
return nil, errwrap.Wrap(err, "error preparing encryption")
}
defer func() {
if outerr != nil {
_ = armoredWriter.Close()
}
}()
_, name := path.Split(s.file)
encWriter, err := openpgp.Encrypt(armoredWriter, entityList, nil, nil, &openpgp.FileHints{
FileName: name,
}, nil)
if err != nil {
return nil, err
}
return &closeAllWriter{
Writer: encWriter,
closers: []io.Closer{encWriter, armoredWriter},
}, nil
})
}
func (s *script) doEncrypt(
extension string,
encryptor func(ciphertextWriter io.Writer) (io.WriteCloser, error),
) (outerr error) {
encFile := fmt.Sprintf("%s.%s", s.file, extension)
s.registerHook(hookLevelPlumbing, func(error) error { s.registerHook(hookLevelPlumbing, func(error) error {
if err := remove(gpgFile); err != nil { if err := remove(encFile); err != nil {
return errwrap.Wrap(err, "error removing gpg file") return errwrap.Wrap(err, "error removing encrypted file")
} }
s.logger.Info( s.logger.Info(
fmt.Sprintf("Removed GPG file `%s`.", gpgFile), fmt.Sprintf("Removed encrypted file `%s`.", encFile),
) )
return nil return nil
}) })
outFile, err := os.Create(gpgFile) outFile, err := os.Create(encFile)
if err != nil { if err != nil {
return errwrap.Wrap(err, "error opening out file") return errwrap.Wrap(err, "error opening out file")
} }
defer func() { defer func() {
if err := outFile.Close(); err != nil { if err := outFile.Close(); err != nil {
cleanUpErr = errors.Join(cleanUpErr, errwrap.Wrap(err, "error closing out file")) outerr = errors.Join(outerr, errwrap.Wrap(err, "error closing out file"))
} }
}() }()
dst, dstCloseCallback, err := encrypt(outFile) dst, err := encryptor(outFile)
if err != nil { if err != nil {
return errwrap.Wrap(err, "error encrypting backup file") return errwrap.Wrap(err, "error encrypting backup file")
} }
defer func() { defer func() {
if err := dstCloseCallback(); err != nil { if err := dst.Close(); err != nil {
cleanUpErr = errors.Join(cleanUpErr, errwrap.Wrap(err, "error closing encrypted backup file")) outerr = errors.Join(outerr, errwrap.Wrap(err, "error closing encrypted backup file"))
} }
}() }()
src, err := os.Open(s.file) src, err := os.Open(s.file)
if err != nil { if err != nil {
return errwrap.Wrap(err, fmt.Sprintf("error opening backup file `%s`", s.file)) return errwrap.Wrap(err, fmt.Sprintf("error opening backup file %q", s.file))
} }
defer func() { defer func() {
if err := src.Close(); err != nil { if err := src.Close(); err != nil {
cleanUpErr = errors.Join(cleanUpErr, errwrap.Wrap(err, "error closing backup file")) outerr = errors.Join(outerr, errwrap.Wrap(err, "error closing backup file"))
} }
}() }()
@@ -121,9 +203,10 @@ func (s *script) encryptArchive() error {
return errwrap.Wrap(err, "error writing ciphertext to file") return errwrap.Wrap(err, "error writing ciphertext to file")
} }
s.file = gpgFile s.file = encFile
s.logger.Info( s.logger.Info(
fmt.Sprintf("Encrypted backup using gpg, saving as `%s`.", s.file), fmt.Sprintf("Encrypted backup using %q, saving as %q", extension, s.file),
) )
return cleanUpErr
return
} }

View File

@@ -199,7 +199,7 @@ func (s *script) init() error {
Endpoint: s.c.AzureStorageEndpoint, Endpoint: s.c.AzureStorageEndpoint,
RemotePath: s.c.AzureStoragePath, RemotePath: s.c.AzureStoragePath,
ConnectionString: s.c.AzureStorageConnectionString, ConnectionString: s.c.AzureStorageConnectionString,
AccessTier: s.c.AzureStorageAccessTier.AccessTier(), AccessTier: s.c.AzureStorageAccessTier,
} }
azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
if err != nil { if err != nil {

View File

@@ -3,15 +3,7 @@ title: Encrypt backups using GPG
layout: default layout: default
parent: How Tos parent: How Tos
nav_order: 7 nav_order: 7
nav_exclude: true
--- ---
# Encrypt backups using GPG See: [Encrypt Backups](encrypt-backups)
The image supports encrypting backups using GPG out of the box.
In case a `GPG_PASSPHRASE` or `GPG_PUBLIC_KEY_RING` environment variable is set, the backup archive will be encrypted using the given key and saved as a `.gpg` file instead.
Assuming you have `gpg` installed, you can decrypt such a backup using (your OS will prompt for the passphrase before decryption can happen):
```console
gpg -o backup.tar.gz -d backup.tar.gz.gpg
```

View File

@@ -0,0 +1,28 @@
---
title: Encrypting backups
layout: default
parent: How Tos
nav_order: 7
---
# Encrypting backups
The image supports encrypting backups using one of two available methods: **GPG** or **[age](https://age-encryption.org/)**
## Using GPG encryption
In case a `GPG_PASSPHRASE` or `GPG_PUBLIC_KEY_RING` environment variable is set, the backup archive will be encrypted using the given key and saved as a `.gpg` file instead.
Assuming you have `gpg` installed, you can decrypt such a backup using (your OS will prompt for the passphrase before decryption can happen):
```console
gpg -o backup.tar.gz -d backup.tar.gz.gpg
```
## Using age encryption
age allows backups to be encrypted with either a symmetric key (password) or a public key. One of those options are available for use.
Given `AGE_PASSPHRASE` being provided, the backup archive will be encrypted with the passphrase and saved as a `.age` file instead. Refer to age documentation for how to properly decrypt.
Given `AGE_PUBLIC_KEYS` being provided (allowing multiple by separating each public key with `,`), the backup archive will be encrypted with the provided public keys. It will also result in the archive being saved as a `.age` file.

3
go.mod
View File

@@ -3,6 +3,7 @@ module github.com/offen/docker-volume-backup
go 1.22 go 1.22
require ( require (
filippo.io/age v1.2.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
github.com/containrrr/shoutrrr v0.8.0 github.com/containrrr/shoutrrr v0.8.0
@@ -13,7 +14,7 @@ require (
github.com/joho/godotenv v1.5.1 github.com/joho/godotenv v1.5.1
github.com/klauspost/compress v1.17.9 github.com/klauspost/compress v1.17.9
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
github.com/minio/minio-go/v7 v7.0.74 github.com/minio/minio-go/v7 v7.0.75
github.com/offen/envconfig v1.5.0 github.com/offen/envconfig v1.5.0
github.com/otiai10/copy v1.14.0 github.com/otiai10/copy v1.14.0
github.com/pkg/sftp v1.13.6 github.com/pkg/sftp v1.13.6

12
go.sum
View File

@@ -1,3 +1,5 @@
c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3IqwfuN5kgDfo5MLzpNM0=
c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805/go.mod h1:FomMrUJ2Lxt5jCLmZkG3FHa72zUprnhd3v/Z18Snm4w=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -31,6 +33,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
filippo.io/age v1.2.0 h1:vRDp7pUMaAJzXNIWJVAZnEf/Dyi4Vu4wI8S1LBzufhE=
filippo.io/age v1.2.0/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
@@ -208,8 +212,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= github.com/minio/minio-go/v7 v7.0.75 h1:0uLrB6u6teY2Jt+cJUVi9cTvDRuBKWSRzSAcznRkwlE=
github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/minio/minio-go/v7 v7.0.75/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI=
@@ -485,8 +489,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@@ -39,11 +39,15 @@ type Config struct {
ConnectionString string ConnectionString string
Endpoint string Endpoint string
RemotePath string RemotePath string
AccessTier *blob.AccessTier AccessTier string
} }
// NewStorageBackend creates and initializes a new Azure Blob Storage backend. // NewStorageBackend creates and initializes a new Azure Blob Storage backend.
func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
if opts.PrimaryAccountKey != "" && opts.ConnectionString != "" {
return nil, errwrap.Wrap(nil, "using primary account key and connection string are mutually exclusive")
}
endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint) endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint)
if err != nil { if err != nil {
return nil, errwrap.Wrap(err, "error parsing endpoint template") return nil, errwrap.Wrap(err, "error parsing endpoint template")
@@ -81,12 +85,26 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error
} }
} }
var uploadStreamOptions *blockblob.UploadStreamOptions
if opts.AccessTier != "" {
var found bool
for _, t := range blob.PossibleAccessTierValues() {
if string(t) == opts.AccessTier {
found = true
uploadStreamOptions = &blockblob.UploadStreamOptions{
AccessTier: &t,
}
}
}
if !found {
return nil, errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", opts.AccessTier))
}
}
storage := azureBlobStorage{ storage := azureBlobStorage{
client: client, client: client,
uploadStreamOptions: &blockblob.UploadStreamOptions{ uploadStreamOptions: uploadStreamOptions,
AccessTier: opts.AccessTier, containerName: opts.ContainerName,
},
containerName: opts.ContainerName,
StorageBackend: &storage.StorageBackend{ StorageBackend: &storage.StorageBackend{
DestinationPath: opts.RemotePath, DestinationPath: opts.RemotePath,
Log: logFunc, Log: logFunc,

View File

@@ -1,8 +1,10 @@
FROM docker:27-dind FROM docker:27-dind
RUN apk add \ RUN apk add \
age \
coreutils \ coreutils \
curl \ curl \
expect \
gpg \ gpg \
gpg-agent \ gpg-agent \
jq \ jq \

View File

@@ -0,0 +1,24 @@
services:
backup:
image: offen/docker-volume-backup:${TEST_VERSION:-canary}
restart: always
environment:
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
BACKUP_FILENAME: test.tar.gz
BACKUP_LATEST_SYMLINK: test-latest.tar.gz.age
BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
AGE_PASSPHRASE: "Dance.0Tonight.Go.Typical"
volumes:
- ${LOCAL_DIR:-./local}:/archive
- app_data:/backup/app_data:ro
- /var/run/docker.sock:/var/run/docker.sock
offen:
image: offen/offen:latest
labels:
- docker-volume-backup.stop-during-backup=true
volumes:
- app_data:/var/opt/offen
volumes:
app_data:

39
test/age-passphrase/run.sh Executable file
View File

@@ -0,0 +1,39 @@
#!/bin/sh
set -e
cd "$(dirname "$0")"
. ../util.sh
current_test=$(basename "$(pwd)")
export LOCAL_DIR="$(mktemp -d)"
docker compose up -d --quiet-pull
sleep 5
docker compose exec backup backup
expect_running_containers "2"
TMP_DIR=$(mktemp -d)
# complex usage of expect(1) due to age not have a way to programmatically
# provide the passphrase
expect -i <<EOL
spawn age --decrypt -o "$LOCAL_DIR/decrypted.tar.gz" "$LOCAL_DIR/test.tar.gz.age"
expect -exact "Enter passphrase: "
send -- "Dance.0Tonight.Go.Typical\r"
sleep 1
EOL
tar -xf "$LOCAL_DIR/decrypted.tar.gz" -C "$TMP_DIR"
if [ ! -f "$TMP_DIR/backup/app_data/offen.db" ]; then
fail "Could not find expected file in untared archive."
fi
rm -vf "$LOCAL_DIR/decrypted.tar.gz"
pass "Found relevant files in decrypted and untared local backup."
if [ ! -L "$LOCAL_DIR/test-latest.tar.gz.age" ]; then
fail "Could not find local symlink to latest encrypted backup."
fi

1
test/age-publickey/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
pk-*.txt

View File

@@ -0,0 +1,24 @@
services:
backup:
image: offen/docker-volume-backup:${TEST_VERSION:-canary}
restart: always
environment:
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
BACKUP_FILENAME: test.tar.gz
BACKUP_LATEST_SYMLINK: test-latest.tar.gz.age
BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
AGE_PUBLIC_KEYS: "${BACKUP_AGE_PUBLIC_KEYS}"
volumes:
- ${LOCAL_DIR:-./local}:/archive
- app_data:/backup/app_data:ro
- /var/run/docker.sock:/var/run/docker.sock
offen:
image: offen/offen:latest
labels:
- docker-volume-backup.stop-during-backup=true
volumes:
- app_data:/var/opt/offen
volumes:
app_data:

43
test/age-publickey/run.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/sh
set -e
cd "$(dirname "$0")"
. ../util.sh
current_test=$(basename "$(pwd)")
export LOCAL_DIR="$(mktemp -d)"
age-keygen >"$LOCAL_DIR/pk-a.txt"
PK_A="$(grep -E 'public key' <"$LOCAL_DIR/pk-a.txt" | cut -d: -f2 | xargs)"
age-keygen >"$LOCAL_DIR/pk-b.txt"
PK_B="$(grep -E 'public key' <"$LOCAL_DIR/pk-b.txt" | cut -d: -f2 | xargs)"
export BACKUP_AGE_PUBLIC_KEYS="$PK_A,$PK_B"
docker compose up -d --quiet-pull
sleep 5
docker compose exec backup backup
expect_running_containers "2"
do_decrypt() {
TMP_DIR=$(mktemp -d)
age --decrypt -i "$1" -o "$LOCAL_DIR/decrypted.tar.gz" "$LOCAL_DIR/test.tar.gz.age"
tar -xf "$LOCAL_DIR/decrypted.tar.gz" -C "$TMP_DIR"
if [ ! -f "$TMP_DIR/backup/app_data/offen.db" ]; then
fail "Could not find expected file in untared archive."
fi
rm -vf "$LOCAL_DIR/decrypted.tar.gz"
pass "Found relevant files in decrypted and untared local backup."
if [ ! -L "$LOCAL_DIR/test-latest.tar.gz.age" ]; then
fail "Could not find local symlink to latest encrypted backup."
fi
}
do_decrypt "$LOCAL_DIR/pk-a.txt"
do_decrypt "$LOCAL_DIR/pk-b.txt"