Added abstract helper interface for all storage backends (#135)

* Added abstract helper interface and implemented it for all storage backends * Moved storage client initializations also to helper classes * Fixed ssh init issue * Moved script parameter to helper struct to simplify script init. * Created sub modules. Enhanced abstract implementation. * Fixed config issue * Fixed declaration issues. Added config to interface. * Added StorageProviders to unify all backends. * Cleanup, optimizations, comments. * Applied discussed changes. See description. Moved modules to internal packages. Replaced StoragePool with slice. Moved conditional for init of storage backends back to script. * Fix docker build issue * Fixed accidentally removed local copy condition. * Delete .gitignore * Renaming/changes according to review Renamed Init functions and interface. Replaced config object with specific config values. Init func returns interface instead of struct. Removed custom import names where possible. * Fixed auto-complete error. * Combined copy instructions into one layer. * Added logging func for storages. * Introduced logging func for errors too. * Missed an error message * Moved config back to main. Optimized prune stats handling. * Move stats back to main package * Code doc stuff * Apply changes from #136 * Replace name field with function. * Changed receiver names from stg to b. * Renamed LogFuncDef to Log * Removed redundant package name. * Renamed storagePool to storages. * Simplified creation of new storage backend. * Added initialization for storage stats map. * Invert .dockerignore patterns. * Fix package typo
2026-04-23 17:05:35 +02:00 · 2022-08-18 08:52:09 +02:00
parent 4ec88d14dd
commit 279844ccfb
14 changed files with 740 additions and 438 deletions
--- a/internal/storage/local/local.go
+++ b/internal/storage/local/local.go
@@ -0,0 +1,131 @@
+package local
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/offen/docker-volume-backup/internal/utilities"
+)
+
+type localStorage struct {
+	*storage.StorageBackend
+	latestSymlink string
+}
+
+// NewStorageBackend creates and initializes a new local storage backend.
+func NewStorageBackend(archivePath string, latestSymlink string, logFunc storage.Log) storage.Backend {
+	return &localStorage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: archivePath,
+			Log:             logFunc,
+		},
+		latestSymlink: latestSymlink,
+	}
+}
+
+// Name return the name of the storage backend
+func (b *localStorage) Name() string {
+	return "Local"
+}
+
+// Copy copies the given file to the local storage backend.
+func (b *localStorage) Copy(file string) error {
+	if _, err := os.Stat(b.DestinationPath); os.IsNotExist(err) {
+		return nil
+	}
+
+	_, name := path.Split(file)
+
+	if err := utilities.CopyFile(file, path.Join(b.DestinationPath, name)); err != nil {
+		return b.Log(storage.ERROR, b.Name(), "Copy: Error copying file to local archive! %w", err)
+	}
+	b.Log(storage.INFO, b.Name(), "Stored copy of backup `%s` in local archive `%s`.", file, b.DestinationPath)
+
+	if b.latestSymlink != "" {
+		symlink := path.Join(b.DestinationPath, b.latestSymlink)
+		if _, err := os.Lstat(symlink); err == nil {
+			os.Remove(symlink)
+		}
+		if err := os.Symlink(name, symlink); err != nil {
+			return b.Log(storage.ERROR, b.Name(), "Copy: error creating latest symlink! %w", err)
+		}
+		b.Log(storage.INFO, b.Name(), "Created/Updated symlink `%s` for latest backup.", b.latestSymlink)
+	}
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the local storage backend.
+func (b *localStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	globPattern := path.Join(
+		b.DestinationPath,
+		fmt.Sprintf("%s*", pruningPrefix),
+	)
+	globMatches, err := filepath.Glob(globPattern)
+	if err != nil {
+		return nil, b.Log(storage.ERROR, b.Name(),
+			"Prune: Error looking up matching files using pattern %s! %w",
+			globPattern,
+			err,
+		)
+	}
+
+	var candidates []string
+	for _, candidate := range globMatches {
+		fi, err := os.Lstat(candidate)
+		if err != nil {
+			return nil, b.Log(storage.ERROR, b.Name(),
+				"Prune: Error calling Lstat on file %s! %w",
+				candidate,
+				err,
+			)
+		}
+
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			candidates = append(candidates, candidate)
+		}
+	}
+
+	var matches []string
+	for _, candidate := range candidates {
+		fi, err := os.Stat(candidate)
+		if err != nil {
+			return nil, b.Log(storage.ERROR, b.Name(),
+				"Prune: Error calling stat on file %s! %w",
+				candidate,
+				err,
+			)
+		}
+		if fi.ModTime().Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(len(candidates)),
+		Pruned: uint(len(matches)),
+	}
+
+	b.DoPrune(b.Name(), len(matches), len(candidates), "local backup(s)", func() error {
+		var removeErrors []error
+		for _, match := range matches {
+			if err := os.Remove(match); err != nil {
+				removeErrors = append(removeErrors, err)
+			}
+		}
+		if len(removeErrors) != 0 {
+			return b.Log(storage.ERROR, b.Name(),
+				"Prune: %d error(s) deleting local files, starting with: %w",
+				len(removeErrors),
+				utilities.Join(removeErrors...),
+			)
+		}
+		return nil
+	})
+
+	return stats, nil
+}
--- a/internal/storage/s3/s3.go
+++ b/internal/storage/s3/s3.go
@@ -0,0 +1,145 @@
+package s3
+
+import (
+	"context"
+	"errors"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/offen/docker-volume-backup/internal/utilities"
+)
+
+type s3Storage struct {
+	*storage.StorageBackend
+	client       *minio.Client
+	bucket       string
+	storageClass string
+}
+
+// NewStorageBackend creates and initializes a new S3/Minio storage backend.
+func NewStorageBackend(endpoint string, accessKeyId string, secretAccessKey string, iamRoleEndpoint string, endpointProto string, endpointInsecure bool,
+	remotePath string, bucket string, storageClass string, logFunc storage.Log) (storage.Backend, error) {
+
+	var creds *credentials.Credentials
+	if accessKeyId != "" && secretAccessKey != "" {
+		creds = credentials.NewStaticV4(
+			accessKeyId,
+			secretAccessKey,
+			"",
+		)
+	} else if iamRoleEndpoint != "" {
+		creds = credentials.NewIAM(iamRoleEndpoint)
+	} else {
+		return nil, errors.New("newScript: AWS_S3_BUCKET_NAME is defined, but no credentials were provided")
+	}
+
+	options := minio.Options{
+		Creds:  creds,
+		Secure: endpointProto == "https",
+	}
+
+	if endpointInsecure {
+		if !options.Secure {
+			return nil, errors.New("newScript: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
+		}
+
+		transport, err := minio.DefaultTransport(true)
+		if err != nil {
+			return nil, logFunc(storage.ERROR, "S3", "NewScript: failed to create default minio transport")
+		}
+		transport.TLSClientConfig.InsecureSkipVerify = true
+		options.Transport = transport
+	}
+
+	mc, err := minio.New(endpoint, &options)
+	if err != nil {
+		return nil, logFunc(storage.ERROR, "S3", "NewScript: error setting up minio client: %w", err)
+	}
+
+	return &s3Storage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: remotePath,
+			Log:             logFunc,
+		},
+		client:       mc,
+		bucket:       bucket,
+		storageClass: storageClass,
+	}, nil
+}
+
+// Name returns the name of the storage backend
+func (v *s3Storage) Name() string {
+	return "S3"
+}
+
+// Copy copies the given file to the S3/Minio storage backend.
+func (b *s3Storage) Copy(file string) error {
+	_, name := path.Split(file)
+
+	if _, err := b.client.FPutObject(context.Background(), b.bucket, filepath.Join(b.DestinationPath, name), file, minio.PutObjectOptions{
+		ContentType:  "application/tar+gzip",
+		StorageClass: b.storageClass,
+	}); err != nil {
+		errResp := minio.ToErrorResponse(err)
+		return b.Log(storage.ERROR, b.Name(), "Copy: error uploading backup to remote storage: [Message]: '%s', [Code]: %s, [StatusCode]: %d", errResp.Message, errResp.Code, errResp.StatusCode)
+	}
+	b.Log(storage.INFO, b.Name(), "Uploaded a copy of backup `%s` to bucket `%s`.", file, b.bucket)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend.
+func (b *s3Storage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates := b.client.ListObjects(context.Background(), b.bucket, minio.ListObjectsOptions{
+		WithMetadata: true,
+		Prefix:       filepath.Join(b.DestinationPath, pruningPrefix),
+		Recursive:    true,
+	})
+
+	var matches []minio.ObjectInfo
+	var lenCandidates int
+	for candidate := range candidates {
+		lenCandidates++
+		if candidate.Err != nil {
+			return nil, b.Log(storage.ERROR, b.Name(),
+				"Prune: Error looking up candidates from remote storage! %w",
+				candidate.Err,
+			)
+		}
+		if candidate.LastModified.Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(lenCandidates),
+		Pruned: uint(len(matches)),
+	}
+
+	b.DoPrune(b.Name(), len(matches), lenCandidates, "remote backup(s)", func() error {
+		objectsCh := make(chan minio.ObjectInfo)
+		go func() {
+			for _, match := range matches {
+				objectsCh <- match
+			}
+			close(objectsCh)
+		}()
+		errChan := b.client.RemoveObjects(context.Background(), b.bucket, objectsCh, minio.RemoveObjectsOptions{})
+		var removeErrors []error
+		for result := range errChan {
+			if result.Err != nil {
+				removeErrors = append(removeErrors, result.Err)
+			}
+		}
+		if len(removeErrors) != 0 {
+			return utilities.Join(removeErrors...)
+		}
+		return nil
+	})
+
+	return stats, nil
+}
--- a/internal/storage/ssh/ssh.go
+++ b/internal/storage/ssh/ssh.go
@@ -0,0 +1,176 @@
+package ssh
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+type sshStorage struct {
+	*storage.StorageBackend
+	client     *ssh.Client
+	sftpClient *sftp.Client
+	hostName   string
+}
+
+// NewStorageBackend creates and initializes a new SSH storage backend.
+func NewStorageBackend(hostName string, port string, user string, password string, identityFile string, identityPassphrase string, remotePath string,
+	logFunc storage.Log) (storage.Backend, error) {
+
+	var authMethods []ssh.AuthMethod
+
+	if password != "" {
+		authMethods = append(authMethods, ssh.Password(password))
+	}
+
+	if _, err := os.Stat(identityFile); err == nil {
+		key, err := ioutil.ReadFile(identityFile)
+		if err != nil {
+			return nil, errors.New("newScript: error reading the private key")
+		}
+
+		var signer ssh.Signer
+		if identityPassphrase != "" {
+			signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(identityPassphrase))
+			if err != nil {
+				return nil, errors.New("newScript: error parsing the encrypted private key")
+			}
+			authMethods = append(authMethods, ssh.PublicKeys(signer))
+		} else {
+			signer, err = ssh.ParsePrivateKey(key)
+			if err != nil {
+				return nil, errors.New("newScript: error parsing the private key")
+			}
+			authMethods = append(authMethods, ssh.PublicKeys(signer))
+		}
+	}
+
+	sshClientConfig := &ssh.ClientConfig{
+		User:            user,
+		Auth:            authMethods,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	sshClient, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", hostName, port), sshClientConfig)
+
+	if err != nil {
+		return nil, logFunc(storage.ERROR, "SSH", "NewScript: Error creating ssh client! %w", err)
+	}
+	_, _, err = sshClient.SendRequest("keepalive", false, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	sftpClient, err := sftp.NewClient(sshClient)
+	if err != nil {
+		return nil, logFunc(storage.ERROR, "SSH", "NewScript: error creating sftp client! %w", err)
+	}
+
+	return &sshStorage{
+		StorageBackend: &storage.StorageBackend{
+			DestinationPath: remotePath,
+			Log:             logFunc,
+		},
+		client:     sshClient,
+		sftpClient: sftpClient,
+		hostName:   hostName,
+	}, nil
+}
+
+// Name returns the name of the storage backend
+func (b *sshStorage) Name() string {
+	return "SSH"
+}
+
+// Copy copies the given file to the SSH storage backend.
+func (b *sshStorage) Copy(file string) error {
+	source, err := os.Open(file)
+	_, name := path.Split(file)
+	if err != nil {
+		return b.Log(storage.ERROR, b.Name(), "Copy: Error reading the file to be uploaded! %w", err)
+	}
+	defer source.Close()
+
+	destination, err := b.sftpClient.Create(filepath.Join(b.DestinationPath, name))
+	if err != nil {
+		return b.Log(storage.ERROR, b.Name(), "Copy: Error creating file on SSH storage! %w", err)
+	}
+	defer destination.Close()
+
+	chunk := make([]byte, 1000000)
+	for {
+		num, err := source.Read(chunk)
+		if err == io.EOF {
+			tot, err := destination.Write(chunk[:num])
+			if err != nil {
+				return b.Log(storage.ERROR, b.Name(), "Copy: Error uploading the file to SSH storage! %w", err)
+			}
+
+			if tot != len(chunk[:num]) {
+				return b.Log(storage.ERROR, b.Name(), "sshClient: failed to write stream")
+			}
+
+			break
+		}
+
+		if err != nil {
+			return b.Log(storage.ERROR, b.Name(), "Copy: Error uploading the file to SSH storage! %w", err)
+		}
+
+		tot, err := destination.Write(chunk[:num])
+		if err != nil {
+			return b.Log(storage.ERROR, b.Name(), "Copy: Error uploading the file to SSH storage! %w", err)
+		}
+
+		if tot != len(chunk[:num]) {
+			return b.Log(storage.ERROR, b.Name(), "sshClient: failed to write stream")
+		}
+	}
+
+	b.Log(storage.INFO, b.Name(), "Uploaded a copy of backup `%s` to SSH storage '%s' at path '%s'.", file, b.hostName, b.DestinationPath)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the SSH storage backend.
+func (b *sshStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates, err := b.sftpClient.ReadDir(b.DestinationPath)
+	if err != nil {
+		return nil, b.Log(storage.ERROR, b.Name(), "Prune: Error reading directory from SSH storage! %w", err)
+	}
+
+	var matches []string
+	for _, candidate := range candidates {
+		if !strings.HasPrefix(candidate.Name(), pruningPrefix) {
+			continue
+		}
+		if candidate.ModTime().Before(deadline) {
+			matches = append(matches, candidate.Name())
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(len(candidates)),
+		Pruned: uint(len(matches)),
+	}
+
+	b.DoPrune(b.Name(), len(matches), len(candidates), "SSH backup(s)", func() error {
+		for _, match := range matches {
+			if err := b.sftpClient.Remove(filepath.Join(b.DestinationPath, match)); err != nil {
+				return b.Log(storage.ERROR, b.Name(), "Prune: Error removing file from SSH storage! %w", err)
+			}
+		}
+		return nil
+	})
+
+	return stats, nil
+}
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -0,0 +1,58 @@
+package storage
+
+import (
+	"time"
+)
+
+// Backend is an interface for defining functions which all storage providers support.
+type Backend interface {
+	Copy(file string) error
+	Prune(deadline time.Time, pruningPrefix string) (*PruneStats, error)
+	Name() string
+}
+
+// StorageBackend is a generic type of storage. Everything here are common properties of all storage types.
+type StorageBackend struct {
+	DestinationPath string
+	RetentionDays   int
+	Log             Log
+}
+
+type LogType string
+
+const (
+	INFO    LogType = "INFO"
+	WARNING LogType = "WARNING"
+	ERROR   LogType = "ERROR"
+)
+
+type Log func(logType LogType, context string, msg string, params ...interface{}) error
+
+// PruneStats is a wrapper struct for returning stats after pruning
+type PruneStats struct {
+	Total  uint
+	Pruned uint
+}
+
+// DoPrune holds general control flow that applies to any kind of storage.
+// Callers can pass in a thunk that performs the actual deletion of files.
+func (b *StorageBackend) DoPrune(context string, lenMatches, lenCandidates int, description string, doRemoveFiles func() error) error {
+	if lenMatches != 0 && lenMatches != lenCandidates {
+		if err := doRemoveFiles(); err != nil {
+			return err
+		}
+		b.Log(INFO, context,
+			"Pruned %d out of %d %s as their age exceeded the configured retention period of %d days.",
+			lenMatches,
+			lenCandidates,
+			description,
+			b.RetentionDays,
+		)
+	} else if lenMatches != 0 && lenMatches == lenCandidates {
+		b.Log(WARNING, context, "The current configuration would delete all %d existing %s.", lenMatches, description)
+		b.Log(WARNING, context, "Refusing to do so, please check your configuration.")
+	} else {
+		b.Log(INFO, context, "None of %d existing %s were pruned.", lenCandidates, description)
+	}
+	return nil
+}
--- a/internal/storage/webdav/webdav.go
+++ b/internal/storage/webdav/webdav.go
@@ -0,0 +1,108 @@
+package webdav
+
+import (
+	"errors"
+	"io/fs"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/studio-b12/gowebdav"
+)
+
+type webDavStorage struct {
+	*storage.StorageBackend
+	client *gowebdav.Client
+	url    string
+}
+
+// NewStorageBackend creates and initializes a new WebDav storage backend.
+func NewStorageBackend(url string, remotePath string, username string, password string, urlInsecure bool,
+	logFunc storage.Log) (storage.Backend, error) {
+
+	if username == "" || password == "" {
+		return nil, errors.New("newScript: WEBDAV_URL is defined, but no credentials were provided")
+	} else {
+		webdavClient := gowebdav.NewClient(url, username, password)
+
+		if urlInsecure {
+			defaultTransport, ok := http.DefaultTransport.(*http.Transport)
+			if !ok {
+				return nil, errors.New("newScript: unexpected error when asserting type for http.DefaultTransport")
+			}
+			webdavTransport := defaultTransport.Clone()
+			webdavTransport.TLSClientConfig.InsecureSkipVerify = urlInsecure
+			webdavClient.SetTransport(webdavTransport)
+		}
+
+		return &webDavStorage{
+			StorageBackend: &storage.StorageBackend{
+				DestinationPath: remotePath,
+				Log:             logFunc,
+			},
+			client: webdavClient,
+		}, nil
+	}
+}
+
+// Name returns the name of the storage backend
+func (b *webDavStorage) Name() string {
+	return "WebDav"
+}
+
+// Copy copies the given file to the WebDav storage backend.
+func (b *webDavStorage) Copy(file string) error {
+	bytes, err := os.ReadFile(file)
+	_, name := path.Split(file)
+	if err != nil {
+		return b.Log(storage.ERROR, b.Name(), "Copy: Error reading the file to be uploaded! %w", err)
+	}
+	if err := b.client.MkdirAll(b.DestinationPath, 0644); err != nil {
+		return b.Log(storage.ERROR, b.Name(), "Copy: Error creating directory '%s' on WebDAV server! %w", b.DestinationPath, err)
+	}
+	if err := b.client.Write(filepath.Join(b.DestinationPath, name), bytes, 0644); err != nil {
+		return b.Log(storage.ERROR, b.Name(), "Copy: Error uploading the file to WebDAV server! %w", err)
+	}
+	b.Log(storage.INFO, b.Name(), "Uploaded a copy of backup `%s` to WebDAV-URL '%s' at path '%s'.", file, b.url, b.DestinationPath)
+
+	return nil
+}
+
+// Prune rotates away backups according to the configuration and provided deadline for the WebDav storage backend.
+func (b *webDavStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
+	candidates, err := b.client.ReadDir(b.DestinationPath)
+	if err != nil {
+		return nil, b.Log(storage.ERROR, b.Name(), "Prune: Error looking up candidates from remote storage! %w", err)
+	}
+	var matches []fs.FileInfo
+	var lenCandidates int
+	for _, candidate := range candidates {
+		if !strings.HasPrefix(candidate.Name(), pruningPrefix) {
+			continue
+		}
+		lenCandidates++
+		if candidate.ModTime().Before(deadline) {
+			matches = append(matches, candidate)
+		}
+	}
+
+	stats := &storage.PruneStats{
+		Total:  uint(lenCandidates),
+		Pruned: uint(len(matches)),
+	}
+
+	b.DoPrune(b.Name(), len(matches), lenCandidates, "WebDAV backup(s)", func() error {
+		for _, match := range matches {
+			if err := b.client.Remove(filepath.Join(b.DestinationPath, match.Name())); err != nil {
+				return b.Log(storage.ERROR, b.Name(), "Prune: Error removing file from WebDAV storage! %w", err)
+			}
+		}
+		return nil
+	})
+
+	return stats, nil
+}