From 27441b71b6644a5de0e27bee12fc9e0c81431445 Mon Sep 17 00:00:00 2001 From: Ben Baron Date: Mon, 6 Jan 2020 11:13:25 -0500 Subject: [PATCH 1/3] Crash when an release asset doesn't exist Currently, the script crashes whenever a release asset is unable to download (for example a 404 response). This change instead logs the failure and allows the script to continue. No retry logic is enabled, but at least it prevents the crash and allows the backup to complete. Retry logic can be implemented later if wanted. closes https://github.com/josegonzalez/python-github-backup/issues/129 --- bin/github-backup | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/bin/github-backup b/bin/github-backup index efa44a6..e451e6a 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -569,15 +569,27 @@ def download_file(url, path, auth): request.add_header('Accept', 'application/octet-stream') request.add_header('Authorization', 'Basic '.encode('ascii') + auth) opener = build_opener(S3HTTPRedirectHandler) - response = opener.open(request) - chunk_size = 16 * 1024 - with open(path, 'wb') as f: - while True: - chunk = response.read(chunk_size) - if not chunk: - break - f.write(chunk) + try: + response = opener.open(request) + + chunk_size = 16 * 1024 + with open(path, 'wb') as f: + while True: + chunk = response.read(chunk_size) + if not chunk: + break + f.write(chunk) + except HTTPError as exc: + # Gracefully handle 404 responses (and others) when downloading from S3 + log_info('Skipping download of asset {0} due to HTTPError: {1}'.format(url, exc.reason)) + except URLError as e: + # Gracefully hadnle other URL errors + log_info('Skipping download of asset {0} due to URLError: {1}'.format(url, e.reason)) + except socket.error as e: + # Gracefully handle socket errors + # TODO: Implement retry logic + log_info('Skipping download of asset {0} due to socker error: {1}'.format(url, e.strerror)) def get_authenticated_user(args): From 38010d7c3976a9f5571c5a4253f8bcd11eb5455d Mon Sep 17 00:00:00 2001 From: Ben Baron Date: Mon, 6 Jan 2020 13:06:22 -0500 Subject: [PATCH 2/3] Switched log_info to log_warning in download_file --- bin/github-backup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/github-backup b/bin/github-backup index e451e6a..5ca799c 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -582,14 +582,14 @@ def download_file(url, path, auth): f.write(chunk) except HTTPError as exc: # Gracefully handle 404 responses (and others) when downloading from S3 - log_info('Skipping download of asset {0} due to HTTPError: {1}'.format(url, exc.reason)) + log_warning('Skipping download of asset {0} due to HTTPError: {1}'.format(url, exc.reason)) except URLError as e: # Gracefully hadnle other URL errors - log_info('Skipping download of asset {0} due to URLError: {1}'.format(url, e.reason)) + log_warning('Skipping download of asset {0} due to URLError: {1}'.format(url, e.reason)) except socket.error as e: # Gracefully handle socket errors # TODO: Implement retry logic - log_info('Skipping download of asset {0} due to socker error: {1}'.format(url, e.strerror)) + log_warning('Skipping download of asset {0} due to socker error: {1}'.format(url, e.strerror)) def get_authenticated_user(args): From cb0293cbe57938cb0286a3feb7c5b121b4c47959 Mon Sep 17 00:00:00 2001 From: Ben Baron Date: Mon, 6 Jan 2020 14:15:41 -0500 Subject: [PATCH 3/3] Fixed comment typo --- bin/github-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/github-backup b/bin/github-backup index 5ca799c..c10694c 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -584,7 +584,7 @@ def download_file(url, path, auth): # Gracefully handle 404 responses (and others) when downloading from S3 log_warning('Skipping download of asset {0} due to HTTPError: {1}'.format(url, exc.reason)) except URLError as e: - # Gracefully hadnle other URL errors + # Gracefully handle other URL errors log_warning('Skipping download of asset {0} due to URLError: {1}'.format(url, e.reason)) except socket.error as e: # Gracefully handle socket errors