Add new storage backend: Dropbox (#103) (#251)

* Add new storage backend: Dropbox (#103)

* Remove duplicate check

* Add concurrency level for parallel upload to dropbox.

* Fixed some instabilites. Changed default concurrency to 6.

* Added some env config vars to readme. WIP

* Wrap errors for storage backend creation.

* Fixed token issue, added OAuth2 including recipe and docs.

* Readme typo fix

* Test for dropbox integration

* Update info and TOC

* Missed a file

* Docker-compose fix

* Fix endpoint connection

* Fix container names

* Fix log fetching

* Fix log fetching (again)

* Print command output to logs

* Addressing comments part 1

* Address comments part 2

* OpenAPI Mock spec path adjusted
* Dropbox FileMetadata reflection refactored
* NaturalNumber type added

* Add OAuth2 mock server for CI testing

* Fix env name of oauth2 endpoint

* Remove hostname

* Add forgotten change to commit...

* Fix oauth2 endpoint

"Worked on my machine"

* Try again

* Try suggested hostname again

* Fix docker internal DNS resolving issues (as suggested by oauth2 mock docs)

* Add docker network, remove hostname

* Network not external

* Last hostname try

* Add more delay, add oauth2 endpoint log

* Temp CI log output of command even when failing

* Try different config and method

* Add custom server-hostname. Rename test folder to accellerate debugging

* Try that fix again

* Adding quotes

* Port fix attempt

* Try localhost

* Try extra hosts

* Change network mode

* Undo some changes

* Use static IP

* Remove specific IP binding

* Change to default net driver

* Fix static IP

* Squash for revert

* Revert "Squash for revert"

This reverts commit e9b617be9a.

* Actual fix for CI testing from #257

test/dropbox/docker-compose.yml

@@ -0,0 +1,57 @@
+version: '3'
+
+services:
+  openapi_mock:
+    image: muonsoft/openapi-mock
+    environment:
+      OPENAPI_MOCK_USE_EXAMPLES: if_present
+      OPENAPI_MOCK_SPECIFICATION_URL: '/etc/openapi/user_v2.yaml'
+    ports:
+      - 8080:8080
+    volumes:
+      - ./user_v2.yaml:/etc/openapi/user_v2.yaml
+
+  oauth2_mock:
+    image: ghcr.io/navikt/mock-oauth2-server:1.0.0
+    ports:
+      - 8090:8090
+    environment:
+      PORT: 8090
+      JSON_CONFIG_PATH: '/etc/oauth2/config.json'
+    volumes:
+      - ./oauth2_config.json:/etc/oauth2/config.json
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    depends_on:
+      - openapi_mock
+      - oauth2_mock
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      DROPBOX_ENDPOINT: http://openapi_mock:8080
+      DROPBOX_OAUTH2_ENDPOINT: http://oauth2_mock:8090
+      DROPBOX_REFRESH_TOKEN: test
+      DROPBOX_APP_KEY: test
+      DROPBOX_APP_SECRET: test
+      DROPBOX_REMOTE_PATH: /test
+      DROPBOX_CONCURRENCY_LEVEL: 6
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/dropbox/oauth2_config.json

@@ -0,0 +1,37 @@
+{
+   "interactiveLogin": true,
+   "httpServer": "NettyWrapper",
+   "tokenCallbacks": [
+       {
+           "issuerId": "issuer1",
+           "tokenExpiry": 120,
+           "requestMappings": [
+               {
+                   "requestParam": "scope",
+                   "match": "scope1",
+                   "claims": {
+                       "sub": "subByScope",
+                       "aud": [
+                           "audByScope"
+                       ]
+                   }
+               }
+           ]
+       },
+       {
+           "issuerId": "issuer2",
+           "requestMappings": [
+               {
+                   "requestParam": "someparam",
+                   "match": "somevalue",
+                   "claims": {
+                       "sub": "subBySomeParam",
+                       "aud": [
+                           "audBySomeParam"
+                       ]
+                   }
+               }
+           ]
+       }
+   ]
+}

test/dropbox/run.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker compose up -d
+sleep 5
+
+logs=$(docker compose exec -T backup backup)
+
+sleep 5
+
+expect_running_containers "4"
+
+echo "$logs"
+if echo "$logs" | grep -q "ERROR"; then
+  fail "Backup failed, errors reported: $dvb_logs"
+else
+  pass "Backup succeeded, no errors reported."
+fi
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+logs=$(docker compose exec -T backup backup)
+
+echo "$logs"
+if echo "$logs" | grep -q "Refusing to do so, please check your configuration"; then
+  pass "Remote backups have not been deleted."
+else
+  fail "Remote backups would have been deleted: $dvb_logs"
+fi
+
+docker compose down --volumes