mirror of
https://github.com/josegonzalez/python-github-backup.git
synced 2025-12-05 16:18:02 +01:00
Compare commits
54 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dd2b96b172 | ||
|
|
7a589f1e63 | ||
|
|
92c619cd01 | ||
|
|
9a91dd7733 | ||
|
|
6592bd8196 | ||
|
|
e9e3b18512 | ||
|
|
88148b4c95 | ||
|
|
8448add464 | ||
|
|
5b30b7ebdd | ||
|
|
c3a17710d3 | ||
|
|
4462412ec7 | ||
|
|
8d61538e5e | ||
|
|
4d37ad206f | ||
|
|
1f983863fc | ||
|
|
f0b28567b9 | ||
|
|
77ede50b19 | ||
|
|
97e4fbbacb | ||
|
|
03604cc654 | ||
|
|
73a62fdee1 | ||
|
|
94e1d62ad5 | ||
|
|
54cef11ce7 | ||
|
|
56397eba1c | ||
|
|
9f861efccf | ||
|
|
c1c9ce6dca | ||
|
|
ab18d8aee0 | ||
|
|
9d7d98b19e | ||
|
|
0233bff696 | ||
|
|
6154ceda15 | ||
|
|
9023052e9c | ||
|
|
874c235ba5 | ||
|
|
b7b234d8a5 | ||
|
|
ed160eb0ca | ||
|
|
1d11d62b73 | ||
|
|
9e1cba9817 | ||
|
|
3859a80b7a | ||
|
|
8c12d54898 | ||
|
|
b6b6605acd | ||
|
|
ff5e0aa89c | ||
|
|
79726c360d | ||
|
|
a511bb2b49 | ||
|
|
aedf9b2c66 | ||
|
|
b9e35a50f5 | ||
|
|
d0e239b3ef | ||
|
|
29c9373d9d | ||
|
|
eb8b22c81c | ||
|
|
03739ce1be | ||
|
|
d2bb205b4b | ||
|
|
17141c1bb6 | ||
|
|
d362adbbca | ||
|
|
1e5a90486c | ||
|
|
9b74aff20b | ||
|
|
89df625e04 | ||
|
|
675484a215 | ||
|
|
325f77dcd9 |
123
CHANGES.rst
123
CHANGES.rst
@@ -1,6 +1,129 @@
|
|||||||
Changelog
|
Changelog
|
||||||
=========
|
=========
|
||||||
|
|
||||||
|
0.12.1 (2017-03-27)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Avoid remote branch name churn. [Chris Adams]
|
||||||
|
|
||||||
|
This avoids the backup output having lots of "[new branch]" messages
|
||||||
|
because removing the old remote name removed all of the existing branch
|
||||||
|
references.
|
||||||
|
|
||||||
|
|
||||||
|
- Fix detection of bare git directories. [Andrzej Maczuga]
|
||||||
|
|
||||||
|
0.12.0 (2016-11-22)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
Fix
|
||||||
|
~~~
|
||||||
|
|
||||||
|
- Properly import version from github_backup package. [Jose Diaz-
|
||||||
|
Gonzalez]
|
||||||
|
|
||||||
|
- Support alternate git status output. [Jose Diaz-Gonzalez]
|
||||||
|
|
||||||
|
Other
|
||||||
|
~~~~~
|
||||||
|
|
||||||
|
- Pep8: E501 line too long (83 > 79 characters) [Jose Diaz-Gonzalez]
|
||||||
|
|
||||||
|
- Pep8: E128 continuation line under-indented for visual indent. [Jose
|
||||||
|
Diaz-Gonzalez]
|
||||||
|
|
||||||
|
- Support archivization using bare git clones. [Andrzej Maczuga]
|
||||||
|
|
||||||
|
- Fix typo, 3x. [Terrell Russell]
|
||||||
|
|
||||||
|
0.11.0 (2016-10-26)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Support --token file:///home/user/token.txt (fixes gh-51) [Björn
|
||||||
|
Dahlgren]
|
||||||
|
|
||||||
|
- Fix some linting. [Albert Wang]
|
||||||
|
|
||||||
|
- Fix byte/string conversion for python 3. [Albert Wang]
|
||||||
|
|
||||||
|
- Support python 3. [Albert Wang]
|
||||||
|
|
||||||
|
- Encode special characters in password. [Remi Rampin]
|
||||||
|
|
||||||
|
- Don't pretend program name is "Github Backup" [Remi Rampin]
|
||||||
|
|
||||||
|
- Don't install over insecure connection. [Remi Rampin]
|
||||||
|
|
||||||
|
The git:// protocol is unauthenticated and unencrypted, and no longer advertised by GitHub. Using HTTPS shouldn't impact performance.
|
||||||
|
|
||||||
|
0.10.3 (2016-08-21)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Fixes #29. [Jonas Michel]
|
||||||
|
|
||||||
|
Reporting an error when the user's rate limit is exceeded causes
|
||||||
|
the script to terminate after resuming execution from a rate limit
|
||||||
|
sleep. Instead of generating an explicit error we just want to
|
||||||
|
inform the user that the script is going to sleep until their rate
|
||||||
|
limit count resets.
|
||||||
|
|
||||||
|
|
||||||
|
- Fixes #29. [Jonas Michel]
|
||||||
|
|
||||||
|
The errors list was not being cleared out after resuming a backup
|
||||||
|
from a rate limit sleep. When the backup was resumed, the non-empty
|
||||||
|
errors list caused the backup to quit after the next `retrieve_data`
|
||||||
|
request.
|
||||||
|
|
||||||
|
|
||||||
|
0.10.2 (2016-08-21)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Add a note regarding git version requirement. [Jose Diaz-Gonzalez]
|
||||||
|
|
||||||
|
Closes #37
|
||||||
|
|
||||||
|
0.10.0 (2016-08-18)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Implement incremental updates. [Robert Bradshaw]
|
||||||
|
|
||||||
|
Guarded with an --incremental flag.
|
||||||
|
|
||||||
|
Stores the time of the last update and only downloads issue and
|
||||||
|
pull request data since this time. All other data is relatively
|
||||||
|
small (likely fetched with a single request) and so is simply
|
||||||
|
re-populated from scratch as before.
|
||||||
|
|
||||||
|
|
||||||
|
0.9.0 (2016-03-29)
|
||||||
|
------------------
|
||||||
|
|
||||||
|
- Fix cloning private repos with basic auth or token. [Kazuki Suda]
|
||||||
|
|
||||||
|
0.8.0 (2016-02-14)
|
||||||
|
------------------
|
||||||
|
|
||||||
|
- Don't store issues which are actually pull requests. [Enrico Tröger]
|
||||||
|
|
||||||
|
This prevents storing pull requests twice since the Github API returns
|
||||||
|
pull requests also as issues. Those issues will be skipped but only if
|
||||||
|
retrieving pull requests is requested as well.
|
||||||
|
Closes #23.
|
||||||
|
|
||||||
|
|
||||||
|
0.7.0 (2016-02-02)
|
||||||
|
------------------
|
||||||
|
|
||||||
|
- Softly fail if not able to read hooks. [Albert Wang]
|
||||||
|
|
||||||
|
- Add note about 2-factor auth. [Albert Wang]
|
||||||
|
|
||||||
|
- Make user repository search go through endpoint capable of reading
|
||||||
|
private repositories. [Albert Wang]
|
||||||
|
|
||||||
|
- Prompt for password if only username given. [Alex Hall]
|
||||||
|
|
||||||
0.6.0 (2015-11-10)
|
0.6.0 (2015-11-10)
|
||||||
------------------
|
------------------
|
||||||
|
|
||||||
|
|||||||
41
README.rst
41
README.rst
@@ -4,6 +4,11 @@ github-backup
|
|||||||
|
|
||||||
backup a github user or organization
|
backup a github user or organization
|
||||||
|
|
||||||
|
Requirements
|
||||||
|
============
|
||||||
|
|
||||||
|
- GIT 1.9+
|
||||||
|
|
||||||
Installation
|
Installation
|
||||||
============
|
============
|
||||||
|
|
||||||
@@ -13,25 +18,25 @@ Using PIP via PyPI::
|
|||||||
|
|
||||||
Using PIP via Github::
|
Using PIP via Github::
|
||||||
|
|
||||||
pip install git+git://github.com/josegonzalez/python-github-backup.git#egg=github-backup
|
pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
|
||||||
|
|
||||||
Usage
|
Usage
|
||||||
=====
|
=====
|
||||||
|
|
||||||
CLI Usage is as follows::
|
CLI Usage is as follows::
|
||||||
|
|
||||||
Github Backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
|
github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
|
||||||
[-o OUTPUT_DIRECTORY] [--starred] [--watched] [--all]
|
[-o OUTPUT_DIRECTORY] [-i] [--starred] [--watched]
|
||||||
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
[--all] [--issues] [--issue-comments] [--issue-events]
|
||||||
[--pull-comments] [--pull-commits] [--repositories]
|
[--pulls] [--pull-comments] [--pull-commits] [--labels]
|
||||||
[--wikis] [--labels] [--hooks] [--skip-existing]
|
[--hooks] [--milestones] [--repositories] [--bare]
|
||||||
|
[--wikis] [--skip-existing]
|
||||||
[-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
|
[-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
|
||||||
[-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
|
[-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
|
||||||
[--prefer-ssh] [-v]
|
[--prefer-ssh] [-v]
|
||||||
USER
|
USER
|
||||||
|
|
||||||
|
Backup a github account
|
||||||
Backup a github users account
|
|
||||||
|
|
||||||
positional arguments:
|
positional arguments:
|
||||||
USER github username
|
USER github username
|
||||||
@@ -41,11 +46,13 @@ CLI Usage is as follows::
|
|||||||
-u USERNAME, --username USERNAME
|
-u USERNAME, --username USERNAME
|
||||||
username for basic auth
|
username for basic auth
|
||||||
-p PASSWORD, --password PASSWORD
|
-p PASSWORD, --password PASSWORD
|
||||||
password for basic auth
|
password for basic auth. If a username is given but
|
||||||
|
not a password, the password will be prompted for.
|
||||||
-t TOKEN, --token TOKEN
|
-t TOKEN, --token TOKEN
|
||||||
personal access or OAuth token
|
personal access or OAuth token
|
||||||
-o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
|
-o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
|
||||||
directory at which to backup the repositories
|
directory at which to backup the repositories
|
||||||
|
-i, --incremental incremental backup
|
||||||
--starred include starred repositories in backup
|
--starred include starred repositories in backup
|
||||||
--watched include watched repositories in backup
|
--watched include watched repositories in backup
|
||||||
--all include everything in backup
|
--all include everything in backup
|
||||||
@@ -55,10 +62,13 @@ CLI Usage is as follows::
|
|||||||
--pulls include pull requests in backup
|
--pulls include pull requests in backup
|
||||||
--pull-comments include pull request review comments in backup
|
--pull-comments include pull request review comments in backup
|
||||||
--pull-commits include pull request commits in backup
|
--pull-commits include pull request commits in backup
|
||||||
--repositories include repository clone in backup
|
|
||||||
--wikis include wiki clone in backup
|
|
||||||
--labels include labels in backup
|
--labels include labels in backup
|
||||||
--hooks include web hooks in backup (works only when authenticated)
|
--hooks include hooks in backup (works only when
|
||||||
|
authenticated)
|
||||||
|
--milestones include milestones in backup
|
||||||
|
--repositories include repository clone in backup
|
||||||
|
--bare clone bare repositories
|
||||||
|
--wikis include wiki clone in backup
|
||||||
--skip-existing skip project if a backup directory exists
|
--skip-existing skip project if a backup directory exists
|
||||||
-L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
|
-L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
|
||||||
only allow these languages
|
only allow these languages
|
||||||
@@ -66,7 +76,7 @@ CLI Usage is as follows::
|
|||||||
python regex to match names against
|
python regex to match names against
|
||||||
-H GITHUB_HOST, --github-host GITHUB_HOST
|
-H GITHUB_HOST, --github-host GITHUB_HOST
|
||||||
GitHub Enterprise hostname
|
GitHub Enterprise hostname
|
||||||
-O, --organization whether or not this is a query for an organization
|
-O, --organization whether or not this is an organization user
|
||||||
-R REPOSITORY, --repository REPOSITORY
|
-R REPOSITORY, --repository REPOSITORY
|
||||||
name of repository to limit backup to
|
name of repository to limit backup to
|
||||||
-P, --private include private repositories
|
-P, --private include private repositories
|
||||||
@@ -76,3 +86,8 @@ CLI Usage is as follows::
|
|||||||
|
|
||||||
|
|
||||||
The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
|
The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
|
||||||
|
|
||||||
|
Authentication
|
||||||
|
==============
|
||||||
|
|
||||||
|
Note: Password-based authentication will fail if you have two-factor authentication enabled.
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import base64
|
|||||||
import calendar
|
import calendar
|
||||||
import codecs
|
import codecs
|
||||||
import errno
|
import errno
|
||||||
|
import getpass
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
@@ -15,8 +16,22 @@ import select
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib
|
try:
|
||||||
import urllib2
|
# python 3
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from urllib.parse import quote as urlquote
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
from urllib.error import HTTPError, URLError
|
||||||
|
from urllib.request import urlopen
|
||||||
|
from urllib.request import Request
|
||||||
|
except ImportError:
|
||||||
|
# python 2
|
||||||
|
from urlparse import urlparse
|
||||||
|
from urllib import quote as urlquote
|
||||||
|
from urllib import urlencode
|
||||||
|
from urllib2 import HTTPError, URLError
|
||||||
|
from urllib2 import urlopen
|
||||||
|
from urllib2 import Request
|
||||||
|
|
||||||
from github_backup import __version__
|
from github_backup import __version__
|
||||||
|
|
||||||
@@ -78,8 +93,8 @@ def logging_subprocess(popenargs,
|
|||||||
rc = child.wait()
|
rc = child.wait()
|
||||||
|
|
||||||
if rc != 0:
|
if rc != 0:
|
||||||
print(u'{} returned {}:'.format(popenargs[0], rc), file=sys.stderr)
|
print('{} returned {}:'.format(popenargs[0], rc), file=sys.stderr)
|
||||||
print('\t', u' '.join(popenargs), file=sys.stderr)
|
print('\t', ' '.join(popenargs), file=sys.stderr)
|
||||||
|
|
||||||
return rc
|
return rc
|
||||||
|
|
||||||
@@ -95,9 +110,19 @@ def mkdir_p(*args):
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def mask_password(url, secret='*****'):
|
||||||
|
parsed = urlparse(url)
|
||||||
|
|
||||||
|
if not parsed.password:
|
||||||
|
return url
|
||||||
|
elif parsed.password == 'x-oauth-basic':
|
||||||
|
return url.replace(parsed.username, secret)
|
||||||
|
|
||||||
|
return url.replace(parsed.password, secret)
|
||||||
|
|
||||||
|
|
||||||
def parse_args():
|
def parse_args():
|
||||||
parser = argparse.ArgumentParser(description='Backup a github account',
|
parser = argparse.ArgumentParser(description='Backup a github account')
|
||||||
prog='Github Backup')
|
|
||||||
parser.add_argument('user',
|
parser.add_argument('user',
|
||||||
metavar='USER',
|
metavar='USER',
|
||||||
type=str,
|
type=str,
|
||||||
@@ -109,16 +134,23 @@ def parse_args():
|
|||||||
parser.add_argument('-p',
|
parser.add_argument('-p',
|
||||||
'--password',
|
'--password',
|
||||||
dest='password',
|
dest='password',
|
||||||
help='password for basic auth')
|
help='password for basic auth. '
|
||||||
|
'If a username is given but not a password, the '
|
||||||
|
'password will be prompted for.')
|
||||||
parser.add_argument('-t',
|
parser.add_argument('-t',
|
||||||
'--token',
|
'--token',
|
||||||
dest='token',
|
dest='token',
|
||||||
help='personal access or OAuth token')
|
help='personal access or OAuth token, or path to token (file://...)') # noqa
|
||||||
parser.add_argument('-o',
|
parser.add_argument('-o',
|
||||||
'--output-directory',
|
'--output-directory',
|
||||||
default='.',
|
default='.',
|
||||||
dest='output_directory',
|
dest='output_directory',
|
||||||
help='directory at which to backup the repositories')
|
help='directory at which to backup the repositories')
|
||||||
|
parser.add_argument('-i',
|
||||||
|
'--incremental',
|
||||||
|
action='store_true',
|
||||||
|
dest='incremental',
|
||||||
|
help='incremental backup')
|
||||||
parser.add_argument('--starred',
|
parser.add_argument('--starred',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_starred',
|
dest='include_starred',
|
||||||
@@ -162,7 +194,7 @@ def parse_args():
|
|||||||
parser.add_argument('--hooks',
|
parser.add_argument('--hooks',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_hooks',
|
dest='include_hooks',
|
||||||
help='include hooks in backup (works only when authenticated)')
|
help='include hooks in backup (works only when authenticated)') # noqa
|
||||||
parser.add_argument('--milestones',
|
parser.add_argument('--milestones',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_milestones',
|
dest='include_milestones',
|
||||||
@@ -171,6 +203,10 @@ def parse_args():
|
|||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_repository',
|
dest='include_repository',
|
||||||
help='include repository clone in backup')
|
help='include repository clone in backup')
|
||||||
|
parser.add_argument('--bare',
|
||||||
|
action='store_true',
|
||||||
|
dest='bare_clone',
|
||||||
|
help='clone bare repositories')
|
||||||
parser.add_argument('--wikis',
|
parser.add_argument('--wikis',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_wiki',
|
dest='include_wiki',
|
||||||
@@ -218,19 +254,34 @@ def parse_args():
|
|||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
def get_auth(args):
|
def get_auth(args, encode=True):
|
||||||
auth = None
|
auth = None
|
||||||
|
|
||||||
if args.token:
|
if args.token:
|
||||||
auth = base64.b64encode(args.token + ':' + 'x-oauth-basic')
|
_path_specifier = 'file://'
|
||||||
elif args.username and args.password:
|
if args.token.startswith(_path_specifier):
|
||||||
auth = base64.b64encode(args.username + ':' + args.password)
|
args.token = open(args.token[len(_path_specifier):],
|
||||||
elif args.username and not args.password:
|
'rt').readline().strip()
|
||||||
log_error('You must specify a password for basic auth')
|
auth = args.token + ':' + 'x-oauth-basic'
|
||||||
elif args.password and not args.username:
|
elif args.username:
|
||||||
|
if not args.password:
|
||||||
|
args.password = getpass.getpass()
|
||||||
|
if encode:
|
||||||
|
password = args.password
|
||||||
|
else:
|
||||||
|
password = urlquote(args.password)
|
||||||
|
auth = args.username + ':' + password
|
||||||
|
elif args.password:
|
||||||
log_error('You must specify a username for basic auth')
|
log_error('You must specify a username for basic auth')
|
||||||
|
|
||||||
|
if not auth:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not encode:
|
||||||
return auth
|
return auth
|
||||||
|
|
||||||
|
return base64.b64encode(auth.encode('ascii'))
|
||||||
|
|
||||||
|
|
||||||
def get_github_api_host(args):
|
def get_github_api_host(args):
|
||||||
if args.github_host:
|
if args.github_host:
|
||||||
@@ -241,7 +292,7 @@ def get_github_api_host(args):
|
|||||||
return host
|
return host
|
||||||
|
|
||||||
|
|
||||||
def get_github_ssh_host(args):
|
def get_github_host(args):
|
||||||
if args.github_host:
|
if args.github_host:
|
||||||
host = args.github_host
|
host = args.github_host
|
||||||
else:
|
else:
|
||||||
@@ -250,6 +301,23 @@ def get_github_ssh_host(args):
|
|||||||
return host
|
return host
|
||||||
|
|
||||||
|
|
||||||
|
def get_github_repo_url(args, repository):
|
||||||
|
if args.prefer_ssh:
|
||||||
|
return repository['ssh_url']
|
||||||
|
|
||||||
|
auth = get_auth(args, False)
|
||||||
|
if auth:
|
||||||
|
repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
|
||||||
|
auth,
|
||||||
|
get_github_host(args),
|
||||||
|
args.user,
|
||||||
|
repository['name'])
|
||||||
|
else:
|
||||||
|
repo_url = repository['clone_url']
|
||||||
|
|
||||||
|
return repo_url
|
||||||
|
|
||||||
|
|
||||||
def retrieve_data(args, template, query_args=None, single_request=False):
|
def retrieve_data(args, template, query_args=None, single_request=False):
|
||||||
auth = get_auth(args)
|
auth = get_auth(args)
|
||||||
query_args = get_query_args(query_args)
|
query_args = get_query_args(query_args)
|
||||||
@@ -269,7 +337,7 @@ def retrieve_data(args, template, query_args=None, single_request=False):
|
|||||||
errors.append(template.format(status_code, r.reason))
|
errors.append(template.format(status_code, r.reason))
|
||||||
log_error(errors)
|
log_error(errors)
|
||||||
|
|
||||||
response = json.loads(r.read())
|
response = json.loads(r.read().decode('utf-8'))
|
||||||
if len(errors) == 0:
|
if len(errors) == 0:
|
||||||
if type(response) == list:
|
if type(response) == list:
|
||||||
data.extend(response)
|
data.extend(response)
|
||||||
@@ -301,11 +369,11 @@ def _get_response(request, auth, template):
|
|||||||
while True:
|
while True:
|
||||||
should_continue = False
|
should_continue = False
|
||||||
try:
|
try:
|
||||||
r = urllib2.urlopen(request)
|
r = urlopen(request)
|
||||||
except urllib2.HTTPError as exc:
|
except HTTPError as exc:
|
||||||
errors, should_continue = _request_http_error(exc, auth, errors) # noqa
|
errors, should_continue = _request_http_error(exc, auth, errors) # noqa
|
||||||
r = exc
|
r = exc
|
||||||
except urllib2.URLError:
|
except URLError:
|
||||||
should_continue = _request_url_error(template, retry_timeout)
|
should_continue = _request_url_error(template, retry_timeout)
|
||||||
if not should_continue:
|
if not should_continue:
|
||||||
raise
|
raise
|
||||||
@@ -318,14 +386,14 @@ def _get_response(request, auth, template):
|
|||||||
|
|
||||||
|
|
||||||
def _construct_request(per_page, page, query_args, template, auth):
|
def _construct_request(per_page, page, query_args, template, auth):
|
||||||
querystring = urllib.urlencode(dict({
|
querystring = urlencode(dict(list({
|
||||||
'per_page': per_page,
|
'per_page': per_page,
|
||||||
'page': page
|
'page': page
|
||||||
}.items() + query_args.items()))
|
}.items()) + list(query_args.items())))
|
||||||
|
|
||||||
request = urllib2.Request(template + '?' + querystring)
|
request = Request(template + '?' + querystring)
|
||||||
if auth is not None:
|
if auth is not None:
|
||||||
request.add_header('Authorization', 'Basic ' + auth)
|
request.add_header('Authorization', 'Basic '.encode('ascii') + auth)
|
||||||
return request
|
return request
|
||||||
|
|
||||||
|
|
||||||
@@ -352,10 +420,9 @@ def _request_http_error(exc, auth, errors):
|
|||||||
print('Exceeded rate limit of {} requests; waiting {} seconds to reset'.format(limit, delta), # noqa
|
print('Exceeded rate limit of {} requests; waiting {} seconds to reset'.format(limit, delta), # noqa
|
||||||
file=sys.stderr)
|
file=sys.stderr)
|
||||||
|
|
||||||
ratelimit_error = 'No more requests remaining'
|
|
||||||
if auth is None:
|
if auth is None:
|
||||||
ratelimit_error += '; authenticate to raise your GitHub rate limit' # noqa
|
print('Hint: Authenticate to raise your GitHub rate limit',
|
||||||
errors.append(ratelimit_error)
|
file=sys.stderr)
|
||||||
|
|
||||||
time.sleep(delta)
|
time.sleep(delta)
|
||||||
should_continue = True
|
should_continue = True
|
||||||
@@ -378,9 +445,8 @@ def _request_url_error(template, retry_timeout):
|
|||||||
def retrieve_repositories(args):
|
def retrieve_repositories(args):
|
||||||
log_info('Retrieving repositories')
|
log_info('Retrieving repositories')
|
||||||
single_request = False
|
single_request = False
|
||||||
template = 'https://{0}/users/{1}/repos'.format(
|
template = 'https://{0}/user/repos'.format(
|
||||||
get_github_api_host(args),
|
get_github_api_host(args))
|
||||||
args.user)
|
|
||||||
if args.organization:
|
if args.organization:
|
||||||
template = 'https://{0}/orgs/{1}/repos'.format(
|
template = 'https://{0}/orgs/{1}/repos'.format(
|
||||||
get_github_api_host(args),
|
get_github_api_host(args),
|
||||||
@@ -396,8 +462,14 @@ def retrieve_repositories(args):
|
|||||||
return retrieve_data(args, template, single_request=single_request)
|
return retrieve_data(args, template, single_request=single_request)
|
||||||
|
|
||||||
|
|
||||||
def filter_repositories(args, repositories):
|
def filter_repositories(args, unfiltered_repositories):
|
||||||
log_info('Filtering repositories')
|
log_info('Filtering repositories')
|
||||||
|
|
||||||
|
repositories = []
|
||||||
|
for r in unfiltered_repositories:
|
||||||
|
if r['owner']['login'] == args.user:
|
||||||
|
repositories.append(r)
|
||||||
|
|
||||||
name_regex = None
|
name_regex = None
|
||||||
if args.name_regex:
|
if args.name_regex:
|
||||||
name_regex = re.compile(args.name_regex)
|
name_regex = re.compile(args.name_regex)
|
||||||
@@ -422,28 +494,36 @@ def backup_repositories(args, output_directory, repositories):
|
|||||||
log_info('Backing up repositories')
|
log_info('Backing up repositories')
|
||||||
repos_template = 'https://{0}/repos'.format(get_github_api_host(args))
|
repos_template = 'https://{0}/repos'.format(get_github_api_host(args))
|
||||||
|
|
||||||
|
if args.incremental:
|
||||||
|
last_update = max(repository['updated_at'] for repository in repositories) # noqa
|
||||||
|
last_update_path = os.path.join(output_directory, 'last_update')
|
||||||
|
if os.path.exists(last_update_path):
|
||||||
|
args.since = open(last_update_path).read().strip()
|
||||||
|
else:
|
||||||
|
args.since = None
|
||||||
|
else:
|
||||||
|
args.since = None
|
||||||
|
|
||||||
for repository in repositories:
|
for repository in repositories:
|
||||||
backup_cwd = os.path.join(output_directory, 'repositories')
|
backup_cwd = os.path.join(output_directory, 'repositories')
|
||||||
repo_cwd = os.path.join(backup_cwd, repository['name'])
|
repo_cwd = os.path.join(backup_cwd, repository['name'])
|
||||||
repo_dir = os.path.join(repo_cwd, 'repository')
|
repo_dir = os.path.join(repo_cwd, 'repository')
|
||||||
|
repo_url = get_github_repo_url(args, repository)
|
||||||
if args.prefer_ssh:
|
|
||||||
repo_url = repository['ssh_url']
|
|
||||||
else:
|
|
||||||
repo_url = repository['clone_url']
|
|
||||||
|
|
||||||
if args.include_repository or args.include_everything:
|
if args.include_repository or args.include_everything:
|
||||||
fetch_repository(repository['name'],
|
fetch_repository(repository['name'],
|
||||||
repo_url,
|
repo_url,
|
||||||
repo_dir,
|
repo_dir,
|
||||||
skip_existing=args.skip_existing)
|
skip_existing=args.skip_existing,
|
||||||
|
bare_clone=args.bare_clone)
|
||||||
|
|
||||||
download_wiki = (args.include_wiki or args.include_everything)
|
download_wiki = (args.include_wiki or args.include_everything)
|
||||||
if repository['has_wiki'] and download_wiki:
|
if repository['has_wiki'] and download_wiki:
|
||||||
fetch_repository(repository['name'],
|
fetch_repository(repository['name'],
|
||||||
repo_url.replace('.git', '.wiki.git'),
|
repo_url.replace('.git', '.wiki.git'),
|
||||||
os.path.join(repo_cwd, 'wiki'),
|
os.path.join(repo_cwd, 'wiki'),
|
||||||
skip_existing=args.skip_existing)
|
skip_existing=args.skip_existing,
|
||||||
|
bare_clone=args.bare_clone)
|
||||||
|
|
||||||
if args.include_issues or args.include_everything:
|
if args.include_issues or args.include_everything:
|
||||||
backup_issues(args, repo_cwd, repository, repos_template)
|
backup_issues(args, repo_cwd, repository, repos_template)
|
||||||
@@ -460,6 +540,9 @@ def backup_repositories(args, output_directory, repositories):
|
|||||||
if args.include_hooks or args.include_everything:
|
if args.include_hooks or args.include_everything:
|
||||||
backup_hooks(args, repo_cwd, repository, repos_template)
|
backup_hooks(args, repo_cwd, repository, repos_template)
|
||||||
|
|
||||||
|
if args.incremental:
|
||||||
|
open(last_update_path, 'w').write(last_update)
|
||||||
|
|
||||||
|
|
||||||
def backup_issues(args, repo_cwd, repository, repos_template):
|
def backup_issues(args, repo_cwd, repository, repos_template):
|
||||||
has_issues_dir = os.path.isdir('{0}/issues/.git'.format(repo_cwd))
|
has_issues_dir = os.path.isdir('{0}/issues/.git'.format(repo_cwd))
|
||||||
@@ -471,26 +554,42 @@ def backup_issues(args, repo_cwd, repository, repos_template):
|
|||||||
mkdir_p(repo_cwd, issue_cwd)
|
mkdir_p(repo_cwd, issue_cwd)
|
||||||
|
|
||||||
issues = {}
|
issues = {}
|
||||||
|
issues_skipped = 0
|
||||||
|
issues_skipped_message = ''
|
||||||
_issue_template = '{0}/{1}/issues'.format(repos_template,
|
_issue_template = '{0}/{1}/issues'.format(repos_template,
|
||||||
repository['full_name'])
|
repository['full_name'])
|
||||||
|
|
||||||
|
should_include_pulls = args.include_pulls or args.include_everything
|
||||||
issue_states = ['open', 'closed']
|
issue_states = ['open', 'closed']
|
||||||
for issue_state in issue_states:
|
for issue_state in issue_states:
|
||||||
query_args = {
|
query_args = {
|
||||||
'filter': 'all',
|
'filter': 'all',
|
||||||
'state': issue_state
|
'state': issue_state
|
||||||
}
|
}
|
||||||
|
if args.since:
|
||||||
|
query_args['since'] = args.since
|
||||||
|
|
||||||
_issues = retrieve_data(args,
|
_issues = retrieve_data(args,
|
||||||
_issue_template,
|
_issue_template,
|
||||||
query_args=query_args)
|
query_args=query_args)
|
||||||
for issue in _issues:
|
for issue in _issues:
|
||||||
|
# skip pull requests which are also returned as issues
|
||||||
|
# if retrieving pull requests is requested as well
|
||||||
|
if 'pull_request' in issue and should_include_pulls:
|
||||||
|
issues_skipped += 1
|
||||||
|
continue
|
||||||
|
|
||||||
issues[issue['number']] = issue
|
issues[issue['number']] = issue
|
||||||
|
|
||||||
log_info('Saving {0} issues to disk'.format(len(issues.keys())))
|
if issues_skipped:
|
||||||
|
issues_skipped_message = ' (skipped {0} pull requests)'.format(
|
||||||
|
issues_skipped)
|
||||||
|
|
||||||
|
log_info('Saving {0} issues to disk{1}'.format(
|
||||||
|
len(list(issues.keys())), issues_skipped_message))
|
||||||
comments_template = _issue_template + '/{0}/comments'
|
comments_template = _issue_template + '/{0}/comments'
|
||||||
events_template = _issue_template + '/{0}/events'
|
events_template = _issue_template + '/{0}/events'
|
||||||
for number, issue in issues.iteritems():
|
for number, issue in list(issues.items()):
|
||||||
if args.include_issue_comments or args.include_everything:
|
if args.include_issue_comments or args.include_everything:
|
||||||
template = comments_template.format(number)
|
template = comments_template.format(number)
|
||||||
issues[number]['comment_data'] = retrieve_data(args, template)
|
issues[number]['comment_data'] = retrieve_data(args, template)
|
||||||
@@ -520,19 +619,24 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
|
|||||||
for pull_state in pull_states:
|
for pull_state in pull_states:
|
||||||
query_args = {
|
query_args = {
|
||||||
'filter': 'all',
|
'filter': 'all',
|
||||||
'state': pull_state
|
'state': pull_state,
|
||||||
|
'sort': 'updated',
|
||||||
|
'direction': 'desc',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# It'd be nice to be able to apply the args.since filter here...
|
||||||
_pulls = retrieve_data(args,
|
_pulls = retrieve_data(args,
|
||||||
_pulls_template,
|
_pulls_template,
|
||||||
query_args=query_args)
|
query_args=query_args)
|
||||||
for pull in _pulls:
|
for pull in _pulls:
|
||||||
|
if not args.since or pull['updated_at'] >= args.since:
|
||||||
pulls[pull['number']] = pull
|
pulls[pull['number']] = pull
|
||||||
|
|
||||||
log_info('Saving {0} pull requests to disk'.format(len(pulls.keys())))
|
log_info('Saving {0} pull requests to disk'.format(
|
||||||
|
len(list(pulls.keys()))))
|
||||||
comments_template = _pulls_template + '/{0}/comments'
|
comments_template = _pulls_template + '/{0}/comments'
|
||||||
commits_template = _pulls_template + '/{0}/commits'
|
commits_template = _pulls_template + '/{0}/commits'
|
||||||
for number, pull in pulls.iteritems():
|
for number, pull in list(pulls.items()):
|
||||||
if args.include_pull_comments or args.include_everything:
|
if args.include_pull_comments or args.include_everything:
|
||||||
template = comments_template.format(number)
|
template = comments_template.format(number)
|
||||||
pulls[number]['comment_data'] = retrieve_data(args, template)
|
pulls[number]['comment_data'] = retrieve_data(args, template)
|
||||||
@@ -566,8 +670,9 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
|
|||||||
for milestone in _milestones:
|
for milestone in _milestones:
|
||||||
milestones[milestone['number']] = milestone
|
milestones[milestone['number']] = milestone
|
||||||
|
|
||||||
log_info('Saving {0} milestones to disk'.format(len(milestones.keys())))
|
log_info('Saving {0} milestones to disk'.format(
|
||||||
for number, milestone in milestones.iteritems():
|
len(list(milestones.keys()))))
|
||||||
|
for number, milestone in list(milestones.items()):
|
||||||
milestone_file = '{0}/{1}.json'.format(milestone_cwd, number)
|
milestone_file = '{0}/{1}.json'.format(milestone_cwd, number)
|
||||||
with codecs.open(milestone_file, 'w', encoding='utf-8') as f:
|
with codecs.open(milestone_file, 'w', encoding='utf-8') as f:
|
||||||
json_dump(milestone, f)
|
json_dump(milestone, f)
|
||||||
@@ -594,39 +699,72 @@ def backup_hooks(args, repo_cwd, repository, repos_template):
|
|||||||
output_file = '{0}/hooks.json'.format(hook_cwd)
|
output_file = '{0}/hooks.json'.format(hook_cwd)
|
||||||
template = '{0}/{1}/hooks'.format(repos_template,
|
template = '{0}/{1}/hooks'.format(repos_template,
|
||||||
repository['full_name'])
|
repository['full_name'])
|
||||||
|
try:
|
||||||
_backup_data(args,
|
_backup_data(args,
|
||||||
'hooks',
|
'hooks',
|
||||||
template,
|
template,
|
||||||
output_file,
|
output_file,
|
||||||
hook_cwd)
|
hook_cwd)
|
||||||
|
except SystemExit:
|
||||||
|
log_info("Unable to read hooks, skipping")
|
||||||
|
|
||||||
|
|
||||||
def fetch_repository(name, remote_url, local_dir, skip_existing=False):
|
def fetch_repository(name,
|
||||||
|
remote_url,
|
||||||
|
local_dir,
|
||||||
|
skip_existing=False,
|
||||||
|
bare_clone=False):
|
||||||
|
if bare_clone:
|
||||||
|
if os.path.exists(local_dir):
|
||||||
|
clone_exists = subprocess.check_output(['git',
|
||||||
|
'rev-parse',
|
||||||
|
'--is-bare-repository'],
|
||||||
|
cwd=local_dir) == "true\n"
|
||||||
|
else:
|
||||||
|
clone_exists = False
|
||||||
|
else:
|
||||||
clone_exists = os.path.exists(os.path.join(local_dir, '.git'))
|
clone_exists = os.path.exists(os.path.join(local_dir, '.git'))
|
||||||
|
|
||||||
if clone_exists and skip_existing:
|
if clone_exists and skip_existing:
|
||||||
return
|
return
|
||||||
|
|
||||||
initalized = subprocess.call('git ls-remote ' + remote_url,
|
masked_remote_url = mask_password(remote_url)
|
||||||
|
|
||||||
|
initialized = subprocess.call('git ls-remote ' + remote_url,
|
||||||
stdout=FNULL,
|
stdout=FNULL,
|
||||||
stderr=FNULL,
|
stderr=FNULL,
|
||||||
shell=True)
|
shell=True)
|
||||||
if initalized == 128:
|
if initialized == 128:
|
||||||
log_info("Skipping {0} ({1}) since it's not initalized".format(name, remote_url))
|
log_info("Skipping {0} ({1}) since it's not initialized".format(
|
||||||
|
name, masked_remote_url))
|
||||||
return
|
return
|
||||||
|
|
||||||
if clone_exists:
|
if clone_exists:
|
||||||
log_info('Updating {0} in {1}'.format(name, local_dir))
|
log_info('Updating {0} in {1}'.format(name, local_dir))
|
||||||
|
|
||||||
|
remotes = subprocess.check_output(['git', 'remote', 'show'],
|
||||||
|
cwd=local_dir)
|
||||||
|
remotes = [i.strip() for i in remotes.decode('utf-8')]
|
||||||
|
|
||||||
|
if 'origin' not in remotes:
|
||||||
git_command = ['git', 'remote', 'rm', 'origin']
|
git_command = ['git', 'remote', 'rm', 'origin']
|
||||||
logging_subprocess(git_command, None, cwd=local_dir)
|
logging_subprocess(git_command, None, cwd=local_dir)
|
||||||
git_command = ['git', 'remote', 'add', 'origin', remote_url]
|
git_command = ['git', 'remote', 'add', 'origin', remote_url]
|
||||||
logging_subprocess(git_command, None, cwd=local_dir)
|
logging_subprocess(git_command, None, cwd=local_dir)
|
||||||
git_command = ['git', 'fetch', '--all', '--tags', '--prune']
|
else:
|
||||||
|
git_command = ['git', 'remote', 'set-url', 'origin', remote_url]
|
||||||
|
logging_subprocess(git_command, None, cwd=local_dir)
|
||||||
|
|
||||||
|
git_command = ['git', 'fetch', '--all', '--force', '--tags', '--prune']
|
||||||
logging_subprocess(git_command, None, cwd=local_dir)
|
logging_subprocess(git_command, None, cwd=local_dir)
|
||||||
else:
|
else:
|
||||||
log_info('Cloning {0} repository from {1} to {2}'.format(name,
|
log_info('Cloning {0} repository from {1} to {2}'.format(
|
||||||
remote_url,
|
name,
|
||||||
|
masked_remote_url,
|
||||||
local_dir))
|
local_dir))
|
||||||
|
if bare_clone:
|
||||||
|
git_command = ['git', 'clone', '--mirror', remote_url, local_dir]
|
||||||
|
else:
|
||||||
git_command = ['git', 'clone', remote_url, local_dir]
|
git_command = ['git', 'clone', remote_url, local_dir]
|
||||||
logging_subprocess(git_command, None)
|
logging_subprocess(git_command, None)
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
__version__ = '0.6.0'
|
__version__ = '0.12.1'
|
||||||
|
|||||||
2
release
2
release
@@ -34,7 +34,7 @@ fi
|
|||||||
echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
|
echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
|
||||||
|
|
||||||
set +e;
|
set +e;
|
||||||
git status | grep "working directory clean" &> /dev/null
|
git status | grep -Eo "working (directory|tree) clean" &> /dev/null
|
||||||
if [ ! $? -eq 0 ]; then # working directory is NOT clean
|
if [ ! $? -eq 0 ]; then # working directory is NOT clean
|
||||||
echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
|
echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
Reference in New Issue
Block a user