Add new storage backend: Dropbox (#103) (#251)

* Add new storage backend: Dropbox (#103)

* Remove duplicate check

* Add concurrency level for parallel upload to dropbox.

* Fixed some instabilites. Changed default concurrency to 6.

* Added some env config vars to readme. WIP

* Wrap errors for storage backend creation.

* Fixed token issue, added OAuth2 including recipe and docs.

* Readme typo fix

* Test for dropbox integration

* Update info and TOC

* Missed a file

* Docker-compose fix

* Fix endpoint connection

* Fix container names

* Fix log fetching

* Fix log fetching (again)

* Print command output to logs

* Addressing comments part 1

* Address comments part 2

* OpenAPI Mock spec path adjusted
* Dropbox FileMetadata reflection refactored
* NaturalNumber type added

* Add OAuth2 mock server for CI testing

* Fix env name of oauth2 endpoint

* Remove hostname

* Add forgotten change to commit...

* Fix oauth2 endpoint

"Worked on my machine"

* Try again

* Try suggested hostname again

* Fix docker internal DNS resolving issues (as suggested by oauth2 mock docs)

* Add docker network, remove hostname

* Network not external

* Last hostname try

* Add more delay, add oauth2 endpoint log

* Temp CI log output of command even when failing

* Try different config and method

* Add custom server-hostname. Rename test folder to accellerate debugging

* Try that fix again

* Adding quotes

* Port fix attempt

* Try localhost

* Try extra hosts

* Change network mode

* Undo some changes

* Use static IP

* Remove specific IP binding

* Change to default net driver

* Fix static IP

* Squash for revert

* Revert "Squash for revert"

This reverts commit e9b617be9a.

* Actual fix for CI testing from #257

cmd/backup/config.go

@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"strconv"
 	"time"
 )
 
@@ -70,6 +71,13 @@ type Config struct {
 	AzureStorageContainerName     string          `split_words:"true"`
 	AzureStoragePath              string          `split_words:"true"`
 	AzureStorageEndpoint          string          `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
+	DropboxEndpoint               string          `split_words:"true" default:"https://api.dropbox.com/"`
+	DropboxOAuth2Endpoint         string          `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"`
+	DropboxRefreshToken           string          `split_words:"true"`
+	DropboxAppKey                 string          `split_words:"true"`
+	DropboxAppSecret              string          `split_words:"true"`
+	DropboxRemotePath             string          `split_words:"true"`
+	DropboxConcurrencyLevel       NaturalNumber   `split_words:"true" default:"6"`
 }
 
 func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
@@ -135,3 +143,21 @@ func (r *RegexpDecoder) Decode(v string) error {
 	*r = RegexpDecoder{Re: re}
 	return nil
 }
+
+type NaturalNumber int
+
+func (n *NaturalNumber) Decode(v string) error {
+	asInt, err := strconv.Atoi(v)
+	if err != nil {
+		return fmt.Errorf("config: error converting %s to int", v)
+	}
+	if asInt <= 0 {
+		return fmt.Errorf("config: expected a natural number, got %d", asInt)
+	}
+	*n = NaturalNumber(asInt)
+	return nil
+}
+
+func (n *NaturalNumber) Int() int {
+	return int(*n)
+}

cmd/backup/script.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/offen/docker-volume-backup/internal/storage"
 	"github.com/offen/docker-volume-backup/internal/storage/azure"
+	"github.com/offen/docker-volume-backup/internal/storage/dropbox"
 	"github.com/offen/docker-volume-backup/internal/storage/local"
 	"github.com/offen/docker-volume-backup/internal/storage/s3"
 	"github.com/offen/docker-volume-backup/internal/storage/ssh"
@@ -70,11 +71,12 @@ func newScript() (*script, error) {
 			StartTime: time.Now(),
 			LogOutput: logBuffer,
 			Storages: map[string]StorageStats{
-				"S3":     {},
-				"WebDAV": {},
-				"SSH":    {},
-				"Local":  {},
-				"Azure":  {},
+				"S3":      {},
+				"WebDAV":  {},
+				"SSH":     {},
+				"Local":   {},
+				"Azure":   {},
+				"Dropbox": {},
 			},
 		},
 	}
@@ -155,7 +157,7 @@ func newScript() (*script, error) {
 			PartSize:         s.c.AwsPartSize,
 		}
 		if s3Backend, err := s3.NewStorageBackend(s3Config, logFunc); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("newScript: error creating s3 storage backend: %w", err)
 		} else {
 			s.storages = append(s.storages, s3Backend)
 		}
@@ -170,7 +172,7 @@ func newScript() (*script, error) {
 			RemotePath:  s.c.WebdavPath,
 		}
 		if webdavBackend, err := webdav.NewStorageBackend(webDavConfig, logFunc); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("newScript: error creating webdav storage backend: %w", err)
 		} else {
 			s.storages = append(s.storages, webdavBackend)
 		}
@@ -187,7 +189,7 @@ func newScript() (*script, error) {
 			RemotePath:         s.c.SSHRemotePath,
 		}
 		if sshBackend, err := ssh.NewStorageBackend(sshConfig, logFunc); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("newScript: error creating ssh storage backend: %w", err)
 		} else {
 			s.storages = append(s.storages, sshBackend)
 		}
@@ -212,11 +214,28 @@ func newScript() (*script, error) {
 		}
 		azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("newScript: error creating azure storage backend: %w", err)
 		}
 		s.storages = append(s.storages, azureBackend)
 	}
 
+	if s.c.DropboxRefreshToken != "" && s.c.DropboxAppKey != "" && s.c.DropboxAppSecret != "" {
+		dropboxConfig := dropbox.Config{
+			Endpoint:         s.c.DropboxEndpoint,
+			OAuth2Endpoint:   s.c.DropboxOAuth2Endpoint,
+			RefreshToken:     s.c.DropboxRefreshToken,
+			AppKey:           s.c.DropboxAppKey,
+			AppSecret:        s.c.DropboxAppSecret,
+			RemotePath:       s.c.DropboxRemotePath,
+			ConcurrencyLevel: s.c.DropboxConcurrencyLevel.Int(),
+		}
+		dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: error creating dropbox storage backend: %w", err)
+		}
+		s.storages = append(s.storages, dropboxBackend)
+	}
+
 	if s.c.EmailNotificationRecipient != "" {
 		emailURL := fmt.Sprintf(
 			"smtp://%s:%s@%s:%d/?from=%s&to=%s",