Compare commits

..

1 Commits

Author SHA1 Message Date
Frederik Ring
f4138fd733 Run httpd server in container if requested 2023-08-04 14:30:47 +02:00
17 changed files with 152 additions and 13575 deletions

View File

@@ -1,7 +1,7 @@
# Copyright 2021 - Offen Authors <hioffen@posteo.de>
# SPDX-License-Identifier: MPL-2.0
FROM golang:1.21-alpine as builder
FROM golang:1.20-alpine as builder
WORKDIR /app
COPY . .
@@ -13,7 +13,7 @@ FROM alpine:3.18
WORKDIR /root
RUN apk add --no-cache ca-certificates
RUN apk add --no-cache ca-certificates busybox-extras
COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
COPY --chmod=755 ./entrypoint.sh /root/

View File

@@ -4,10 +4,10 @@
# docker-volume-backup
Backup Docker volumes locally or to any S3, WebDAV, Azure Blob Storage, Dropbox or SSH compatible storage.
Backup Docker volumes locally or to any S3, WebDAV, Azure Blob Storage or SSH compatible storage.
The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup.
It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV, Azure Blob Storage, Dropbox or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__.
It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV, Azure Blob Storage or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__.
<!-- MarkdownTOC -->
@@ -36,7 +36,6 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
- [Define different retention schedules](#define-different-retention-schedules)
- [Use special characters in notification URLs](#use-special-characters-in-notification-urls)
- [Handle file uploads using third party tools](#handle-file-uploads-using-third-party-tools)
- [Setup Dropbox storage backend](#setup-dropbox-storage-backend)
- [Recipes](#recipes)
- [Backing up to AWS S3](#backing-up-to-aws-s3)
- [Backing up to Filebase](#backing-up-to-filebase)
@@ -45,7 +44,6 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
- [Backing up to WebDAV](#backing-up-to-webdav)
- [Backing up to SSH](#backing-up-to-ssh)
- [Backing up to Azure Blob Storage](#backing-up-to-azure-blob-storage)
- [Backing up to Dropbox](#backing-up-to-dropbox)
- [Backing up locally](#backing-up-locally)
- [Backing up to AWS S3 as well as locally](#backing-up-to-aws-s3-as-well-as-locally)
- [Running on a custom cron schedule](#running-on-a-custom-cron-schedule)
@@ -150,22 +148,13 @@ You can populate below template according to your requirements and use it as you
# BACKUP_CRON_EXPRESSION="0 2 * * *"
# The compression algorithm used in conjunction with tar.
# Valid options are: "gz" (Gzip) and "zst" (Zstd).
# Note that the selection affects the file extension.
# BACKUP_COMPRESSION="gz"
# The name of the backup file including the extension.
# The name of the backup file including the `.tar.gz` extension.
# Format verbs will be replaced as in `strftime`. Omitting them
# will result in the same filename for every backup run, which means previous
# versions will be overwritten on subsequent runs.
# Extension can be defined literally or via "{{ .Extension }}" template,
# in which case it will become either "tar.gz" or "tar.zst" (depending
# on your BACKUP_COMPRESSION setting).
# The default results in filenames like: `backup-2021-08-29T04-00-00.tar.gz`.
# versions will be overwritten on subsequent runs. The default results
# in filenames like `backup-2021-08-29T04-00-00.tar.gz`.
# BACKUP_FILENAME="backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
# BACKUP_FILENAME="backup-%Y-%m-%dT%H-%M-%S.tar.gz"
# Setting BACKUP_FILENAME_EXPAND to true allows for environment variable
# placeholders in BACKUP_FILENAME, BACKUP_LATEST_SYMLINK and in
@@ -358,26 +347,6 @@ You can populate below template according to your requirements and use it as you
# AZURE_STORAGE_ENDPOINT="https://{{ .AccountName }}.blob.core.windows.net/"
# Absolute remote path in your Dropbox where the backups shall be stored.
# Note: Use your app's subpath in Dropbox, if it doesn't have global access.
# Consulte the README for further information.
# DROPBOX_REMOTE_PATH="/my/directory"
# Number of concurrent chunked uploads for Dropbox.
# Values above 6 usually result in no enhancements.
# DROPBOX_CONCURRENCY_LEVEL="6"
# App key and app secret from your app created at https://www.dropbox.com/developers/apps/info
# DROPBOX_APP_KEY=""
# DROPBOX_APP_SECRET=""
# Refresh token to request new short-lived tokens (OAuth2). Consult README to see how to get one.
# DROPBOX_REFRESH_TOKEN=""
# In addition to storing backups remotely, you can also keep local copies.
# Pass a container-local path to store your backups if needed. You also need to
# mount a local folder or Docker volume into that location (`/archive`
@@ -1042,37 +1011,6 @@ volumes:
Commands will be invoked with the filepath of the tar archive passed as `COMMAND_RUNTIME_BACKUP_FILEPATH`.
### Setup Dropbox storage backend
#### Auth-Setup:
1. Create a new Dropbox App in the [App Console](https://www.dropbox.com/developers/apps)
2. Open your new Dropbox App and set `DROPBOX_APP_KEY` and `DROPBOX_APP_SECRET` in your environment (e.g. docker-compose.yml) accordingly
3. Click on `Permissions` in your app and make sure, that the following permissions are cranted (or more):
- `files.metadata.write`
- `files.metadata.read`
- `files.content.write`
- `files.content.read`
4. Replace APPKEY in `https://www.dropbox.com/oauth2/authorize?client_id=APPKEY&token_access_type=offline&response_type=code` with the app key from step 2
5. Visit the URL and confirm the access of your app. This gives you an `auth code` -> save it somewhere!
6. Replace AUTHCODE, APPKEY, APPSECRET accordingly and perform the request:
```
curl https://api.dropbox.com/oauth2/token \
-d code=AUTHCODE \
-d grant_type=authorization_code \
-d client_id=APPKEY \
-d client_secret=APPSECRET
```
7. Execute the request. You will get a JSON formatted reply. Use the value of the `refresh_token` for the last environment variable `DROPBOX_REFRESH_TOKEN`
8. You should now have `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET` and `DROPBOX_REFRESH_TOKEN` set. These don't expire.
Note: Using the "Generated access token" in the app console is not supported, as it is only very short lived and therefore not suitable for an automatic backup solution. The refresh token handles this automatically - the setup procedure above is only needed once.
#### Other parameters
Important: If you chose `App folder` access during the creation of your Dropbox app in step 1 above, you can only write in the app's directory!
This means, that `DROPBOX_REMOTE_PATH` must start with e.g. `/Apps/YOUR_APP_NAME` or `/Apps/YOUR_APP_NAME/some_sub_dir`
## Recipes
This section lists configuration for some real-world use cases that you can mix and match according to your needs.
@@ -1240,30 +1178,6 @@ volumes:
data:
```
### Backing up to Dropbox
See [Dropbox Setup](#setup-dropbox-storage-backend) on how to get the appropriate environment values.
```yml
version: '3'
services:
# ... define other services using the `data` volume here
backup:
image: offen/docker-volume-backup:v2
environment:
DROPBOX_REFRESH_TOKEN: REFRESH_KEY # replace
DROPBOX_APP_KEY: APP_KEY # replace
DROPBOX_APP_SECRET: APP_SECRET # replace
DROPBOX_REMOTE_PATH: /Apps/my-test-app/some_subdir # replace
volumes:
- data:/backup/my-app-backup:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
volumes:
data:
```
### Backing up locally
```yml

View File

@@ -15,11 +15,9 @@ import (
"path"
"path/filepath"
"strings"
"github.com/klauspost/compress/zstd"
)
func createArchive(files []string, inputFilePath, outputFilePath string, compression string) error {
func createArchive(files []string, inputFilePath, outputFilePath string) error {
inputFilePath = stripTrailingSlashes(inputFilePath)
inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
if err != nil {
@@ -29,7 +27,7 @@ func createArchive(files []string, inputFilePath, outputFilePath string, compres
return fmt.Errorf("createArchive: error creating output file path: %w", err)
}
if err := compress(files, outputFilePath, filepath.Dir(inputFilePath), compression); err != nil {
if err := compress(files, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
return fmt.Errorf("createArchive: error creating archive: %w", err)
}
@@ -53,30 +51,18 @@ func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error)
return inputFilePath, outputFilePath, err
}
func compress(paths []string, outFilePath, subPath string, algo string) error {
func compress(paths []string, outFilePath, subPath string) error {
file, err := os.Create(outFilePath)
var compressWriter io.WriteCloser
if err != nil {
return fmt.Errorf("compress: error creating out file: %w", err)
}
prefix := path.Dir(outFilePath)
switch algo {
case "gz":
compressWriter = gzip.NewWriter(file)
case "zst":
compressWriter, err = zstd.NewWriter(file)
if err != nil {
return fmt.Errorf("compress: zstd error: %w", err)
}
default:
return fmt.Errorf("compress: unsupported compression algorithm: %s", algo)
}
tarWriter := tar.NewWriter(compressWriter)
gzipWriter := gzip.NewWriter(file)
tarWriter := tar.NewWriter(gzipWriter)
for _, p := range paths {
if err := writeTarball(p, tarWriter, prefix); err != nil {
if err := writeTarGz(p, tarWriter, prefix); err != nil {
return fmt.Errorf("compress: error writing %s to archive: %w", p, err)
}
}
@@ -86,9 +72,9 @@ func compress(paths []string, outFilePath, subPath string, algo string) error {
return fmt.Errorf("compress: error closing tar writer: %w", err)
}
err = compressWriter.Close()
err = gzipWriter.Close()
if err != nil {
return fmt.Errorf("compress: error closing compression writer: %w", err)
return fmt.Errorf("compress: error closing gzip writer: %w", err)
}
err = file.Close()
@@ -99,10 +85,10 @@ func compress(paths []string, outFilePath, subPath string, algo string) error {
return nil
}
func writeTarball(path string, tarWriter *tar.Writer, prefix string) error {
func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
fileInfo, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("writeTarball: error getting file infor for %s: %w", path, err)
return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err)
}
if fileInfo.Mode()&os.ModeSocket == os.ModeSocket {
@@ -113,19 +99,19 @@ func writeTarball(path string, tarWriter *tar.Writer, prefix string) error {
if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
var err error
if link, err = os.Readlink(path); err != nil {
return fmt.Errorf("writeTarball: error resolving symlink %s: %w", path, err)
return fmt.Errorf("writeTarGz: error resolving symlink %s: %w", path, err)
}
}
header, err := tar.FileInfoHeader(fileInfo, link)
if err != nil {
return fmt.Errorf("writeTarball: error getting file info header: %w", err)
return fmt.Errorf("writeTarGz: error getting file info header: %w", err)
}
header.Name = strings.TrimPrefix(path, prefix)
err = tarWriter.WriteHeader(header)
if err != nil {
return fmt.Errorf("writeTarball: error writing file info header: %w", err)
return fmt.Errorf("writeTarGz: error writing file info header: %w", err)
}
if !fileInfo.Mode().IsRegular() {
@@ -134,13 +120,13 @@ func writeTarball(path string, tarWriter *tar.Writer, prefix string) error {
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("writeTarball: error opening %s: %w", path, err)
return fmt.Errorf("writeTarGz: error opening %s: %w", path, err)
}
defer file.Close()
_, err = io.Copy(tarWriter, file)
if err != nil {
return fmt.Errorf("writeTarball: error copying %s to tar writer: %w", path, err)
return fmt.Errorf("writeTarGz: error copying %s to tar writer: %w", path, err)
}
return nil

View File

@@ -10,74 +10,65 @@ import (
"io/ioutil"
"os"
"regexp"
"strconv"
"time"
)
// Config holds all configuration values that are expected to be set
// by users.
type Config struct {
AwsS3BucketName string `split_words:"true"`
AwsS3Path string `split_words:"true"`
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
AwsEndpointProto string `split_words:"true" default:"https"`
AwsEndpointInsecure bool `split_words:"true"`
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
AwsStorageClass string `split_words:"true"`
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
AwsSecretAccessKey string `split_words:"true"`
AwsSecretAccessKeyFile string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"`
AwsPartSize int64 `split_words:"true"`
BackupCompression CompressionType `split_words:"true" default:"gz"`
BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"`
BackupFilenameExpand bool `split_words:"true"`
BackupLatestSymlink string `split_words:"true"`
BackupArchive string `split_words:"true" default:"/archive"`
BackupRetentionDays int32 `split_words:"true" default:"-1"`
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
BackupPruningPrefix string `split_words:"true"`
BackupStopContainerLabel string `split_words:"true" default:"true"`
BackupFromSnapshot bool `split_words:"true"`
BackupExcludeRegexp RegexpDecoder `split_words:"true"`
GpgPassphrase string `split_words:"true"`
NotificationURLs []string `envconfig:"NOTIFICATION_URLS"`
NotificationLevel string `split_words:"true" default:"error"`
EmailNotificationRecipient string `split_words:"true"`
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
WebdavUrl string `split_words:"true"`
WebdavUrlInsecure bool `split_words:"true"`
WebdavPath string `split_words:"true" default:"/"`
WebdavUsername string `split_words:"true"`
WebdavPassword string `split_words:"true"`
SSHHostName string `split_words:"true"`
SSHPort string `split_words:"true" default:"22"`
SSHUser string `split_words:"true"`
SSHPassword string `split_words:"true"`
SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"`
SSHIdentityPassphrase string `split_words:"true"`
SSHRemotePath string `split_words:"true"`
ExecLabel string `split_words:"true"`
ExecForwardOutput bool `split_words:"true"`
LockTimeout time.Duration `split_words:"true" default:"60m"`
AzureStorageAccountName string `split_words:"true"`
AzureStoragePrimaryAccountKey string `split_words:"true"`
AzureStorageContainerName string `split_words:"true"`
AzureStoragePath string `split_words:"true"`
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"`
DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"`
DropboxRefreshToken string `split_words:"true"`
DropboxAppKey string `split_words:"true"`
DropboxAppSecret string `split_words:"true"`
DropboxRemotePath string `split_words:"true"`
DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"`
AwsS3BucketName string `split_words:"true"`
AwsS3Path string `split_words:"true"`
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
AwsEndpointProto string `split_words:"true" default:"https"`
AwsEndpointInsecure bool `split_words:"true"`
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
AwsStorageClass string `split_words:"true"`
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
AwsSecretAccessKey string `split_words:"true"`
AwsSecretAccessKeyFile string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"`
AwsPartSize int64 `split_words:"true"`
BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
BackupFilenameExpand bool `split_words:"true"`
BackupLatestSymlink string `split_words:"true"`
BackupArchive string `split_words:"true" default:"/archive"`
BackupRetentionDays int32 `split_words:"true" default:"-1"`
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
BackupPruningPrefix string `split_words:"true"`
BackupStopContainerLabel string `split_words:"true" default:"true"`
BackupFromSnapshot bool `split_words:"true"`
BackupExcludeRegexp RegexpDecoder `split_words:"true"`
GpgPassphrase string `split_words:"true"`
NotificationURLs []string `envconfig:"NOTIFICATION_URLS"`
NotificationLevel string `split_words:"true" default:"error"`
EmailNotificationRecipient string `split_words:"true"`
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
WebdavUrl string `split_words:"true"`
WebdavUrlInsecure bool `split_words:"true"`
WebdavPath string `split_words:"true" default:"/"`
WebdavUsername string `split_words:"true"`
WebdavPassword string `split_words:"true"`
SSHHostName string `split_words:"true"`
SSHPort string `split_words:"true" default:"22"`
SSHUser string `split_words:"true"`
SSHPassword string `split_words:"true"`
SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"`
SSHIdentityPassphrase string `split_words:"true"`
SSHRemotePath string `split_words:"true"`
ExecLabel string `split_words:"true"`
ExecForwardOutput bool `split_words:"true"`
LockTimeout time.Duration `split_words:"true" default:"60m"`
AzureStorageAccountName string `split_words:"true"`
AzureStoragePrimaryAccountKey string `split_words:"true"`
AzureStorageContainerName string `split_words:"true"`
AzureStoragePath string `split_words:"true"`
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
}
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
@@ -91,22 +82,6 @@ func (c *Config) resolveSecret(envVar string, secretPath string) (string, error)
return string(data), nil
}
type CompressionType string
func (c *CompressionType) Decode(v string) error {
switch v {
case "gz", "zst":
*c = CompressionType(v)
return nil
default:
return fmt.Errorf("config: error decoding compression type %s", v)
}
}
func (c *CompressionType) String() string {
return string(*c)
}
type CertDecoder struct {
Cert *x509.Certificate
}
@@ -143,21 +118,3 @@ func (r *RegexpDecoder) Decode(v string) error {
*r = RegexpDecoder{Re: re}
return nil
}
type NaturalNumber int
func (n *NaturalNumber) Decode(v string) error {
asInt, err := strconv.Atoi(v)
if err != nil {
return fmt.Errorf("config: error converting %s to int", v)
}
if asInt <= 0 {
return fmt.Errorf("config: expected a natural number, got %d", asInt)
}
*n = NaturalNumber(asInt)
return nil
}
func (n *NaturalNumber) Int() int {
return int(*n)
}

View File

@@ -160,7 +160,7 @@ func (s *script) runLabeledCommands(label string) error {
userLabelName := fmt.Sprintf("%s.user", label)
user := c.Labels[userLabelName]
s.logger.Info(fmt.Sprintf("Running %s command %s for container %s", label, cmd, strings.TrimPrefix(c.Names[0], "/")))
s.logger.Infof("Running %s command %s for container %s", label, cmd, strings.TrimPrefix(c.Names[0], "/"))
stdout, stderr, err := s.exec(c.ID, cmd, user)
if s.c.ExecForwardOutput {
os.Stderr.Write(stderr)

View File

@@ -41,11 +41,9 @@ func (s *script) lock(lockfile string) (func() error, error) {
}
if !s.encounteredLock {
s.logger.Info(
fmt.Sprintf(
"Exclusive lock was not available on first attempt. Will retry until it becomes available or the timeout of %s is exceeded.",
s.c.LockTimeout,
),
s.logger.Infof(
"Exclusive lock was not available on first attempt. Will retry until it becomes available or the timeout of %s is exceeded.",
s.c.LockTimeout,
)
s.encounteredLock = true
}

View File

@@ -4,7 +4,6 @@
package main
import (
"fmt"
"os"
)
@@ -22,9 +21,7 @@ func main() {
if pArg := recover(); pArg != nil {
if err, ok := pArg.(error); ok {
if hookErr := s.runHooks(err); hookErr != nil {
s.logger.Error(
fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr),
)
s.logger.Errorf("An error occurred calling the registered hooks: %s", hookErr)
}
os.Exit(1)
}
@@ -32,12 +29,9 @@ func main() {
}
if err := s.runHooks(nil); err != nil {
s.logger.Error(
fmt.Sprintf(
"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
err,
),
s.logger.Errorf(
"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
err,
)
os.Exit(1)
}

View File

@@ -4,13 +4,11 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"os"
"path"
"path/filepath"
@@ -19,7 +17,6 @@ import (
"github.com/offen/docker-volume-backup/internal/storage"
"github.com/offen/docker-volume-backup/internal/storage/azure"
"github.com/offen/docker-volume-backup/internal/storage/dropbox"
"github.com/offen/docker-volume-backup/internal/storage/local"
"github.com/offen/docker-volume-backup/internal/storage/s3"
"github.com/offen/docker-volume-backup/internal/storage/ssh"
@@ -35,6 +32,7 @@ import (
"github.com/kelseyhightower/envconfig"
"github.com/leekchan/timeutil"
"github.com/otiai10/copy"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/openpgp"
"golang.org/x/sync/errgroup"
)
@@ -44,7 +42,7 @@ import (
type script struct {
cli *client.Client
storages []storage.Backend
logger *slog.Logger
logger *logrus.Logger
sender *router.ServiceRouter
template *template.Template
hooks []hook
@@ -65,18 +63,22 @@ type script struct {
func newScript() (*script, error) {
stdOut, logBuffer := buffer(os.Stdout)
s := &script{
c: &Config{},
logger: slog.New(slog.NewTextHandler(stdOut, nil)),
c: &Config{},
logger: &logrus.Logger{
Out: stdOut,
Formatter: new(logrus.TextFormatter),
Hooks: make(logrus.LevelHooks),
Level: logrus.InfoLevel,
},
stats: &Stats{
StartTime: time.Now(),
LogOutput: logBuffer,
Storages: map[string]StorageStats{
"S3": {},
"WebDAV": {},
"SSH": {},
"Local": {},
"Azure": {},
"Dropbox": {},
"S3": {},
"WebDAV": {},
"SSH": {},
"Local": {},
"Azure": {},
},
},
}
@@ -92,20 +94,6 @@ func newScript() (*script, error) {
}
s.file = path.Join("/tmp", s.c.BackupFilename)
tmplFileName, tErr := template.New("extension").Parse(s.file)
if tErr != nil {
return nil, fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr)
}
var bf bytes.Buffer
if tErr := tmplFileName.Execute(&bf, map[string]string{
"Extension": fmt.Sprintf("tar.%s", s.c.BackupCompression),
}); tErr != nil {
return nil, fmt.Errorf("newScript: error executing backup file extension template: %w", tErr)
}
s.file = bf.String()
if s.c.BackupFilenameExpand {
s.file = os.ExpandEnv(s.file)
s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
@@ -126,11 +114,12 @@ func newScript() (*script, error) {
logFunc := func(logType storage.LogLevel, context string, msg string, params ...any) {
switch logType {
case storage.LogLevelWarning:
s.logger.Warn(fmt.Sprintf("["+context+"] "+msg, params...))
s.logger.Warnf("["+context+"] "+msg, params...)
case storage.LogLevelError:
s.logger.Error(fmt.Sprintf("["+context+"] "+msg, params...))
s.logger.Errorf("["+context+"] "+msg, params...)
case storage.LogLevelInfo:
default:
s.logger.Info(fmt.Sprintf("["+context+"] "+msg, params...))
s.logger.Infof("["+context+"] "+msg, params...)
}
}
@@ -157,7 +146,7 @@ func newScript() (*script, error) {
PartSize: s.c.AwsPartSize,
}
if s3Backend, err := s3.NewStorageBackend(s3Config, logFunc); err != nil {
return nil, fmt.Errorf("newScript: error creating s3 storage backend: %w", err)
return nil, err
} else {
s.storages = append(s.storages, s3Backend)
}
@@ -172,7 +161,7 @@ func newScript() (*script, error) {
RemotePath: s.c.WebdavPath,
}
if webdavBackend, err := webdav.NewStorageBackend(webDavConfig, logFunc); err != nil {
return nil, fmt.Errorf("newScript: error creating webdav storage backend: %w", err)
return nil, err
} else {
s.storages = append(s.storages, webdavBackend)
}
@@ -189,7 +178,7 @@ func newScript() (*script, error) {
RemotePath: s.c.SSHRemotePath,
}
if sshBackend, err := ssh.NewStorageBackend(sshConfig, logFunc); err != nil {
return nil, fmt.Errorf("newScript: error creating ssh storage backend: %w", err)
return nil, err
} else {
s.storages = append(s.storages, sshBackend)
}
@@ -214,28 +203,11 @@ func newScript() (*script, error) {
}
azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
if err != nil {
return nil, fmt.Errorf("newScript: error creating azure storage backend: %w", err)
return nil, err
}
s.storages = append(s.storages, azureBackend)
}
if s.c.DropboxRefreshToken != "" && s.c.DropboxAppKey != "" && s.c.DropboxAppSecret != "" {
dropboxConfig := dropbox.Config{
Endpoint: s.c.DropboxEndpoint,
OAuth2Endpoint: s.c.DropboxOAuth2Endpoint,
RefreshToken: s.c.DropboxRefreshToken,
AppKey: s.c.DropboxAppKey,
AppSecret: s.c.DropboxAppSecret,
RemotePath: s.c.DropboxRemotePath,
ConcurrencyLevel: s.c.DropboxConcurrencyLevel.Int(),
}
dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc)
if err != nil {
return nil, fmt.Errorf("newScript: error creating dropbox storage backend: %w", err)
}
s.storages = append(s.storages, dropboxBackend)
}
if s.c.EmailNotificationRecipient != "" {
emailURL := fmt.Sprintf(
"smtp://%s:%s@%s:%d/?from=%s&to=%s",
@@ -334,13 +306,11 @@ func (s *script) stopContainers() (func() error, error) {
return noop, nil
}
s.logger.Info(
fmt.Sprintf(
"Stopping %d container(s) labeled `%s` out of %d running container(s).",
len(containersToStop),
containerLabel,
len(allContainers),
),
s.logger.Infof(
"Stopping %d container(s) labeled `%s` out of %d running container(s).",
len(containersToStop),
containerLabel,
len(allContainers),
)
var stoppedContainers []types.Container
@@ -412,11 +382,9 @@ func (s *script) stopContainers() (func() error, error) {
errors.Join(restartErrors...),
)
}
s.logger.Info(
fmt.Sprintf(
"Restarted %d container(s) and the matching service(s).",
len(stoppedContainers),
),
s.logger.Infof(
"Restarted %d container(s) and the matching service(s).",
len(stoppedContainers),
)
return nil
}, stopError
@@ -440,9 +408,7 @@ func (s *script) createArchive() error {
if err := remove(backupSources); err != nil {
return fmt.Errorf("createArchive: error removing snapshot: %w", err)
}
s.logger.Info(
fmt.Sprintf("Removed snapshot `%s`.", backupSources),
)
s.logger.Infof("Removed snapshot `%s`.", backupSources)
return nil
})
if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
@@ -451,9 +417,7 @@ func (s *script) createArchive() error {
}); err != nil {
return fmt.Errorf("createArchive: error creating snapshot: %w", err)
}
s.logger.Info(
fmt.Sprintf("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources),
)
s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources)
}
tarFile := s.file
@@ -461,9 +425,7 @@ func (s *script) createArchive() error {
if err := remove(tarFile); err != nil {
return fmt.Errorf("createArchive: error removing tar file: %w", err)
}
s.logger.Info(
fmt.Sprintf("Removed tar file `%s`.", tarFile),
)
s.logger.Infof("Removed tar file `%s`.", tarFile)
return nil
})
@@ -487,13 +449,11 @@ func (s *script) createArchive() error {
return fmt.Errorf("createArchive: error walking filesystem tree: %w", err)
}
if err := createArchive(filesEligibleForBackup, backupSources, tarFile, s.c.BackupCompression.String()); err != nil {
if err := createArchive(filesEligibleForBackup, backupSources, tarFile); err != nil {
return fmt.Errorf("createArchive: error compressing backup folder: %w", err)
}
s.logger.Info(
fmt.Sprintf("Created backup of `%s` at `%s`.", backupSources, tarFile),
)
s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, tarFile)
return nil
}
@@ -510,9 +470,7 @@ func (s *script) encryptArchive() error {
if err := remove(gpgFile); err != nil {
return fmt.Errorf("encryptArchive: error removing gpg file: %w", err)
}
s.logger.Info(
fmt.Sprintf("Removed GPG file `%s`.", gpgFile),
)
s.logger.Infof("Removed GPG file `%s`.", gpgFile)
return nil
})
@@ -542,9 +500,7 @@ func (s *script) encryptArchive() error {
}
s.file = gpgFile
s.logger.Info(
fmt.Sprintf("Encrypted backup using given passphrase, saving as `%s`.", s.file),
)
s.logger.Infof("Encrypted backup using given passphrase, saving as `%s`.", s.file)
return nil
}
@@ -616,9 +572,7 @@ func (s *script) pruneBackups() error {
// is non-nil.
func (s *script) must(err error) {
if err != nil {
s.logger.Error(
fmt.Sprintf("Fatal error running backup: %s", err),
)
s.logger.Errorf("Fatal error running backup: %s", err)
panic(err)
}
}

View File

@@ -22,5 +22,12 @@ else
done
fi
if [ ! -z "$SERVE_METRICS_PATH" ]; then
mkdir -p /var/www/html${SERVE_METRICS_PATH}
echo "ok" > /var/www/html${SERVE_METRICS_PATH}/metrics.txt
httpd -h /var/www/html -p "${SERVE_METRICS_PORT:-80}"
echo "Serving metrics on port ${SERVE_METRICS_PORT:-80}."
fi
echo "Starting cron in foreground."
crond -f -d 8

26
go.mod
View File

@@ -1,6 +1,6 @@
module github.com/offen/docker-volume-backup
go 1.21
go 1.19
require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
@@ -10,23 +10,16 @@ require (
github.com/docker/docker v24.0.5+incompatible
github.com/gofrs/flock v0.8.1
github.com/kelseyhightower/envconfig v1.4.0
github.com/klauspost/compress v1.16.7
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
github.com/minio/minio-go/v7 v7.0.62
github.com/minio/minio-go/v7 v7.0.61
github.com/otiai10/copy v1.11.0
github.com/pkg/sftp v1.13.6
github.com/pkg/sftp v1.13.5
github.com/sirupsen/logrus v1.9.3
github.com/studio-b12/gowebdav v0.9.0
golang.org/x/crypto v0.12.0
golang.org/x/crypto v0.11.0
golang.org/x/sync v0.3.0
)
require (
github.com/golang/protobuf v1.5.2 // indirect
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.28.1 // indirect
)
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
@@ -35,13 +28,13 @@ require (
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
@@ -59,10 +52,9 @@ require (
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gotest.tools/v3 v3.0.3 // indirect

35
go.sum
View File

@@ -188,7 +188,6 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9Orh
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
@@ -248,7 +247,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY=
@@ -257,8 +255,6 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU=
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
@@ -494,8 +490,8 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.62 h1:qNYsFZHEzl+NfH8UxW4jpmlKav1qUAgfY30YNRneVhc=
github.com/minio/minio-go/v7 v7.0.62/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI=
github.com/minio/minio-go/v7 v7.0.61/go.mod h1:BTu8FcrEw+HidY0zd/0eny43QnVNkXRPXrLXFuQBHXg=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -547,7 +543,6 @@ github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod
github.com/otiai10/copy v1.11.0 h1:OKBD80J/mLBrwnzXqGtFCzprFSGioo30JcmR4APsNwc=
github.com/otiai10/copy v1.11.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -559,8 +554,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -662,10 +657,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -762,8 +758,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -787,7 +783,6 @@ golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7Lm
golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk=
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -902,13 +897,12 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -919,8 +913,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1049,7 +1043,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=

View File

@@ -1,260 +0,0 @@
package dropbox
import (
"bytes"
"context"
"fmt"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
"github.com/offen/docker-volume-backup/internal/storage"
"golang.org/x/oauth2"
)
type dropboxStorage struct {
*storage.StorageBackend
client files.Client
concurrencyLevel int
}
// Config allows to configure a Dropbox storage backend.
type Config struct {
Endpoint string
OAuth2Endpoint string
RefreshToken string
AppKey string
AppSecret string
RemotePath string
ConcurrencyLevel int
}
// NewStorageBackend creates and initializes a new Dropbox storage backend.
func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
tokenUrl, _ := url.JoinPath(opts.OAuth2Endpoint, "oauth2/token")
conf := &oauth2.Config{
ClientID: opts.AppKey,
ClientSecret: opts.AppSecret,
Endpoint: oauth2.Endpoint{
TokenURL: tokenUrl,
},
}
logFunc(storage.LogLevelInfo, "Dropbox", "Fetching fresh access token for Dropbox storage backend.")
tkSource := conf.TokenSource(context.Background(), &oauth2.Token{RefreshToken: opts.RefreshToken})
token, err := tkSource.Token()
if err != nil {
return nil, fmt.Errorf("(*dropboxStorage).NewStorageBackend: Error refreshing token: %w", err)
}
dbxConfig := dropbox.Config{
Token: token.AccessToken,
}
if opts.Endpoint != "https://api.dropbox.com/" {
dbxConfig.URLGenerator = func(hostType string, namespace string, route string) string {
return fmt.Sprintf("%s/%d/%s/%s", opts.Endpoint, 2, namespace, route)
}
}
client := files.New(dbxConfig)
if opts.ConcurrencyLevel < 1 {
logFunc(storage.LogLevelWarning, "Dropbox", "Concurrency level must be at least 1! Using 1 instead of %d.", opts.ConcurrencyLevel)
opts.ConcurrencyLevel = 1
}
return &dropboxStorage{
StorageBackend: &storage.StorageBackend{
DestinationPath: opts.RemotePath,
Log: logFunc,
},
client: client,
concurrencyLevel: opts.ConcurrencyLevel,
}, nil
}
// Name returns the name of the storage backend
func (b *dropboxStorage) Name() string {
return "Dropbox"
}
// Copy copies the given file to the WebDav storage backend.
func (b *dropboxStorage) Copy(file string) error {
_, name := path.Split(file)
folderArg := files.NewCreateFolderArg(b.DestinationPath)
if _, err := b.client.CreateFolderV2(folderArg); err != nil {
switch err := err.(type) {
case files.CreateFolderV2APIError:
if err.EndpointError.Path.Tag != files.WriteErrorConflict {
return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s' in Dropbox: %w", b.DestinationPath, err)
}
b.Log(storage.LogLevelInfo, b.Name(), "Destination path '%s' already exists in Dropbox, no new directory required.", b.DestinationPath)
default:
return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s' in Dropbox: %w", b.DestinationPath, err)
}
}
r, err := os.Open(file)
if err != nil {
return fmt.Errorf("(*dropboxStorage).Copy: Error opening the file to be uploaded: %w", err)
}
defer r.Close()
// Start new upload session and get session id
b.Log(storage.LogLevelInfo, b.Name(), "Starting upload session for backup '%s' to Dropbox at path '%s'.", file, b.DestinationPath)
var sessionId string
uploadSessionStartArg := files.NewUploadSessionStartArg()
uploadSessionStartArg.SessionType = &files.UploadSessionType{Tagged: dropbox.Tagged{Tag: files.UploadSessionTypeConcurrent}}
if res, err := b.client.UploadSessionStart(uploadSessionStartArg, nil); err != nil {
return fmt.Errorf("(*dropboxStorage).Copy: Error starting the upload session: %w", err)
} else {
sessionId = res.SessionId
}
// Send the file in 148MB chunks (Dropbox API limit is 150MB, concurrent upload requires a multiple of 4MB though)
// Last append can be any size <= 150MB with Close=True
const chunkSize = 148 * 1024 * 1024 // 148MB
var offset uint64 = 0
var guard = make(chan struct{}, b.concurrencyLevel)
var errorChn = make(chan error, b.concurrencyLevel)
var EOFChn = make(chan bool, b.concurrencyLevel)
var mu sync.Mutex
var wg sync.WaitGroup
loop:
for {
guard <- struct{}{} // limit concurrency
select {
case err := <-errorChn: // error from goroutine
return err
case <-EOFChn: // EOF from goroutine
wg.Wait() // wait for all goroutines to finish
break loop
default:
}
go func() {
defer func() {
wg.Done()
<-guard
}()
wg.Add(1)
chunk := make([]byte, chunkSize)
mu.Lock() // to preserve offset of chunks
select {
case <-EOFChn:
EOFChn <- true // put it back for outer loop
mu.Unlock()
return // already EOF
default:
}
bytesRead, err := r.Read(chunk)
if err != nil {
errorChn <- fmt.Errorf("(*dropboxStorage).Copy: Error reading the file to be uploaded: %w", err)
mu.Unlock()
return
}
chunk = chunk[:bytesRead]
uploadSessionAppendArg := files.NewUploadSessionAppendArg(
files.NewUploadSessionCursor(sessionId, offset),
)
isEOF := bytesRead < chunkSize
uploadSessionAppendArg.Close = isEOF
if isEOF {
EOFChn <- true
}
offset += uint64(bytesRead)
mu.Unlock()
if err := b.client.UploadSessionAppendV2(uploadSessionAppendArg, bytes.NewReader(chunk)); err != nil {
errorChn <- fmt.Errorf("(*dropboxStorage).Copy: Error appending the file to the upload session: %w", err)
return
}
}()
}
// Finish the upload session, commit the file (no new data added)
_, err = b.client.UploadSessionFinish(
files.NewUploadSessionFinishArg(
files.NewUploadSessionCursor(sessionId, 0),
files.NewCommitInfo(filepath.Join(b.DestinationPath, name)),
), nil)
if err != nil {
return fmt.Errorf("(*dropboxStorage).Copy: Error finishing the upload session: %w", err)
}
b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup '%s' to Dropbox at path '%s'.", file, b.DestinationPath)
return nil
}
// Prune rotates away backups according to the configuration and provided deadline for the Dropbox storage backend.
func (b *dropboxStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
var entries []files.IsMetadata
res, err := b.client.ListFolder(files.NewListFolderArg(b.DestinationPath))
if err != nil {
return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err)
}
entries = append(entries, res.Entries...)
for res.HasMore {
res, err = b.client.ListFolderContinue(files.NewListFolderContinueArg(res.Cursor))
if err != nil {
return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err)
}
entries = append(entries, res.Entries...)
}
var matches []*files.FileMetadata
var lenCandidates int
for _, candidate := range entries {
switch candidate := candidate.(type) {
case *files.FileMetadata:
if !strings.HasPrefix(candidate.Name, pruningPrefix) {
continue
}
lenCandidates++
if candidate.ServerModified.Before(deadline) {
matches = append(matches, candidate)
}
default:
continue
}
}
stats := &storage.PruneStats{
Total: uint(lenCandidates),
Pruned: uint(len(matches)),
}
if err := b.DoPrune(b.Name(), len(matches), lenCandidates, "Dropbox backup(s)", func() error {
for _, match := range matches {
if _, err := b.client.DeleteV2(files.NewDeleteArg(filepath.Join(b.DestinationPath, match.Name))); err != nil {
return fmt.Errorf("(*dropboxStorage).Prune: Error removing file from Dropbox storage: %w", err)
}
}
return nil
}); err != nil {
return stats, err
}
return stats, nil
}

View File

@@ -1,66 +0,0 @@
#!/bin/sh
set -e
cd $(dirname $0)
. ../util.sh
current_test=$(basename $(pwd))
docker network create test_network
docker volume create backup_data
docker volume create app_data
# This volume is created to test whether empty directories are handled
# correctly. It is not supposed to hold any data.
docker volume create empty_data
docker run -d \
--name minio \
--network test_network \
--env MINIO_ROOT_USER=test \
--env MINIO_ROOT_PASSWORD=test \
--env MINIO_ACCESS_KEY=test \
--env MINIO_SECRET_KEY=GMusLtUmILge2by+z890kQ \
-v backup_data:/data \
minio/minio:RELEASE.2020-08-04T23-10-51Z server /data
docker exec minio mkdir -p /data/backup
docker run -d \
--name offen \
--network test_network \
-v app_data:/var/opt/offen/ \
offen/offen:latest
sleep 10
docker run --rm \
--network test_network \
-v app_data:/backup/app_data \
-v empty_data:/backup/empty_data \
-v /var/run/docker.sock:/var/run/docker.sock \
--env AWS_ACCESS_KEY_ID=test \
--env AWS_SECRET_ACCESS_KEY=GMusLtUmILge2by+z890kQ \
--env AWS_ENDPOINT=minio:9000 \
--env AWS_ENDPOINT_PROTO=http \
--env AWS_S3_BUCKET_NAME=backup \
--env BACKUP_COMPRESSION=zst \
--env BACKUP_FILENAME='test.{{ .Extension }}' \
--env "BACKUP_FROM_SNAPSHOT=true" \
--entrypoint backup \
offen/docker-volume-backup:${TEST_VERSION:-canary}
# Have to install tar and zstd on Alpine because the plain image comes with very
# basic tar from busybox and it does not seem to support zstd
docker run --rm \
-v backup_data:/data alpine \
ash -c 'apk add --no-cache zstd tar && tar -xvf /data/backup/test.tar.zst --zstd && test -f /backup/app_data/offen.db && test -d /backup/empty_data'
pass "Found relevant files in untared remote backup."
# This test does not stop containers during backup. This is happening on
# purpose in order to cover this setup as well.
expect_running_containers "2"
docker rm $(docker stop minio offen)
docker volume rm backup_data app_data
docker network rm test_network

View File

@@ -1,57 +0,0 @@
version: '3'
services:
openapi_mock:
image: muonsoft/openapi-mock
environment:
OPENAPI_MOCK_USE_EXAMPLES: if_present
OPENAPI_MOCK_SPECIFICATION_URL: '/etc/openapi/user_v2.yaml'
ports:
- 8080:8080
volumes:
- ./user_v2.yaml:/etc/openapi/user_v2.yaml
oauth2_mock:
image: ghcr.io/navikt/mock-oauth2-server:1.0.0
ports:
- 8090:8090
environment:
PORT: 8090
JSON_CONFIG_PATH: '/etc/oauth2/config.json'
volumes:
- ./oauth2_config.json:/etc/oauth2/config.json
backup:
image: offen/docker-volume-backup:${TEST_VERSION:-canary}
hostname: hostnametoken
depends_on:
- openapi_mock
- oauth2_mock
restart: always
environment:
BACKUP_FILENAME_EXPAND: 'true'
BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
BACKUP_PRUNING_LEEWAY: 5s
BACKUP_PRUNING_PREFIX: test
DROPBOX_ENDPOINT: http://openapi_mock:8080
DROPBOX_OAUTH2_ENDPOINT: http://oauth2_mock:8090
DROPBOX_REFRESH_TOKEN: test
DROPBOX_APP_KEY: test
DROPBOX_APP_SECRET: test
DROPBOX_REMOTE_PATH: /test
DROPBOX_CONCURRENCY_LEVEL: 6
volumes:
- app_data:/backup/app_data:ro
- /var/run/docker.sock:/var/run/docker.sock
offen:
image: offen/offen:latest
labels:
- docker-volume-backup.stop-during-backup=true
volumes:
- app_data:/var/opt/offen
volumes:
app_data:

View File

@@ -1,37 +0,0 @@
{
"interactiveLogin": true,
"httpServer": "NettyWrapper",
"tokenCallbacks": [
{
"issuerId": "issuer1",
"tokenExpiry": 120,
"requestMappings": [
{
"requestParam": "scope",
"match": "scope1",
"claims": {
"sub": "subByScope",
"aud": [
"audByScope"
]
}
}
]
},
{
"issuerId": "issuer2",
"requestMappings": [
{
"requestParam": "someparam",
"match": "somevalue",
"claims": {
"sub": "subBySomeParam",
"aud": [
"audBySomeParam"
]
}
}
]
}
]
}

View File

@@ -1,40 +0,0 @@
#!/bin/sh
set -e
cd "$(dirname "$0")"
. ../util.sh
current_test=$(basename $(pwd))
docker compose up -d
sleep 5
logs=$(docker compose exec -T backup backup)
sleep 5
expect_running_containers "4"
echo "$logs"
if echo "$logs" | grep -q "ERROR"; then
fail "Backup failed, errors reported: $dvb_logs"
else
pass "Backup succeeded, no errors reported."
fi
# The second part of this test checks if backups get deleted when the retention
# is set to 0 days (which it should not as it would mean all backups get deleted)
# TODO: find out if we can test actual deletion without having to wait for a day
BACKUP_RETENTION_DAYS="0" docker compose up -d
sleep 5
logs=$(docker compose exec -T backup backup)
echo "$logs"
if echo "$logs" | grep -q "Refusing to do so, please check your configuration"; then
pass "Remote backups have not been deleted."
else
fail "Remote backups would have been deleted: $dvb_logs"
fi
docker compose down --volumes

File diff suppressed because it is too large Load Diff