mirror of
https://github.com/josegonzalez/python-github-backup.git
synced 2025-12-05 16:18:02 +01:00
Compare commits
45 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2efeaa7580 | ||
|
|
647810a2f0 | ||
|
|
0dfe5c342a | ||
|
|
1d6e1abab1 | ||
|
|
dd2b96b172 | ||
|
|
7a589f1e63 | ||
|
|
92c619cd01 | ||
|
|
9a91dd7733 | ||
|
|
6592bd8196 | ||
|
|
e9e3b18512 | ||
|
|
88148b4c95 | ||
|
|
8448add464 | ||
|
|
5b30b7ebdd | ||
|
|
c3a17710d3 | ||
|
|
4462412ec7 | ||
|
|
8d61538e5e | ||
|
|
4d37ad206f | ||
|
|
1f983863fc | ||
|
|
f0b28567b9 | ||
|
|
77ede50b19 | ||
|
|
97e4fbbacb | ||
|
|
03604cc654 | ||
|
|
73a62fdee1 | ||
|
|
94e1d62ad5 | ||
|
|
54cef11ce7 | ||
|
|
56397eba1c | ||
|
|
9f861efccf | ||
|
|
c1c9ce6dca | ||
|
|
ab18d8aee0 | ||
|
|
9d7d98b19e | ||
|
|
0233bff696 | ||
|
|
6154ceda15 | ||
|
|
9023052e9c | ||
|
|
874c235ba5 | ||
|
|
b7b234d8a5 | ||
|
|
ed160eb0ca | ||
|
|
1d11d62b73 | ||
|
|
9e1cba9817 | ||
|
|
3859a80b7a | ||
|
|
8c12d54898 | ||
|
|
b6b6605acd | ||
|
|
ff5e0aa89c | ||
|
|
79726c360d | ||
|
|
1e5a90486c | ||
|
|
9b74aff20b |
120
CHANGES.rst
120
CHANGES.rst
@@ -1,6 +1,126 @@
|
|||||||
Changelog
|
Changelog
|
||||||
=========
|
=========
|
||||||
|
|
||||||
|
0.13.0 (2017-04-05)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Add OS check for OSX specific keychain args. [Martin O'Reilly]
|
||||||
|
|
||||||
|
Keychain arguments are only supported on Mac OSX.
|
||||||
|
Added check for operating system so we give a
|
||||||
|
"Keychain arguments are only supported on Mac OSX"
|
||||||
|
error message rather than a "No password item matching the
|
||||||
|
provided name and account could be found in the osx keychain"
|
||||||
|
error message
|
||||||
|
|
||||||
|
|
||||||
|
- Add support for storing PAT in OSX keychain. [Martin O'Reilly]
|
||||||
|
|
||||||
|
Added additional optional arguments and README guidance for storing
|
||||||
|
and accessing a Github personal access token (PAT) in the OSX
|
||||||
|
keychain
|
||||||
|
|
||||||
|
|
||||||
|
0.12.1 (2017-03-27)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Avoid remote branch name churn. [Chris Adams]
|
||||||
|
|
||||||
|
This avoids the backup output having lots of "[new branch]" messages
|
||||||
|
because removing the old remote name removed all of the existing branch
|
||||||
|
references.
|
||||||
|
|
||||||
|
|
||||||
|
- Fix detection of bare git directories. [Andrzej Maczuga]
|
||||||
|
|
||||||
|
0.12.0 (2016-11-22)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
Fix
|
||||||
|
~~~
|
||||||
|
|
||||||
|
- Properly import version from github_backup package. [Jose Diaz-
|
||||||
|
Gonzalez]
|
||||||
|
|
||||||
|
- Support alternate git status output. [Jose Diaz-Gonzalez]
|
||||||
|
|
||||||
|
Other
|
||||||
|
~~~~~
|
||||||
|
|
||||||
|
- Pep8: E501 line too long (83 > 79 characters) [Jose Diaz-Gonzalez]
|
||||||
|
|
||||||
|
- Pep8: E128 continuation line under-indented for visual indent. [Jose
|
||||||
|
Diaz-Gonzalez]
|
||||||
|
|
||||||
|
- Support archivization using bare git clones. [Andrzej Maczuga]
|
||||||
|
|
||||||
|
- Fix typo, 3x. [Terrell Russell]
|
||||||
|
|
||||||
|
0.11.0 (2016-10-26)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Support --token file:///home/user/token.txt (fixes gh-51) [Björn
|
||||||
|
Dahlgren]
|
||||||
|
|
||||||
|
- Fix some linting. [Albert Wang]
|
||||||
|
|
||||||
|
- Fix byte/string conversion for python 3. [Albert Wang]
|
||||||
|
|
||||||
|
- Support python 3. [Albert Wang]
|
||||||
|
|
||||||
|
- Encode special characters in password. [Remi Rampin]
|
||||||
|
|
||||||
|
- Don't pretend program name is "Github Backup" [Remi Rampin]
|
||||||
|
|
||||||
|
- Don't install over insecure connection. [Remi Rampin]
|
||||||
|
|
||||||
|
The git:// protocol is unauthenticated and unencrypted, and no longer advertised by GitHub. Using HTTPS shouldn't impact performance.
|
||||||
|
|
||||||
|
0.10.3 (2016-08-21)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Fixes #29. [Jonas Michel]
|
||||||
|
|
||||||
|
Reporting an error when the user's rate limit is exceeded causes
|
||||||
|
the script to terminate after resuming execution from a rate limit
|
||||||
|
sleep. Instead of generating an explicit error we just want to
|
||||||
|
inform the user that the script is going to sleep until their rate
|
||||||
|
limit count resets.
|
||||||
|
|
||||||
|
|
||||||
|
- Fixes #29. [Jonas Michel]
|
||||||
|
|
||||||
|
The errors list was not being cleared out after resuming a backup
|
||||||
|
from a rate limit sleep. When the backup was resumed, the non-empty
|
||||||
|
errors list caused the backup to quit after the next `retrieve_data`
|
||||||
|
request.
|
||||||
|
|
||||||
|
|
||||||
|
0.10.2 (2016-08-21)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Add a note regarding git version requirement. [Jose Diaz-Gonzalez]
|
||||||
|
|
||||||
|
Closes #37
|
||||||
|
|
||||||
|
0.10.0 (2016-08-18)
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
- Implement incremental updates. [Robert Bradshaw]
|
||||||
|
|
||||||
|
Guarded with an --incremental flag.
|
||||||
|
|
||||||
|
Stores the time of the last update and only downloads issue and
|
||||||
|
pull request data since this time. All other data is relatively
|
||||||
|
small (likely fetched with a single request) and so is simply
|
||||||
|
re-populated from scratch as before.
|
||||||
|
|
||||||
|
|
||||||
|
0.9.0 (2016-03-29)
|
||||||
|
------------------
|
||||||
|
|
||||||
|
- Fix cloning private repos with basic auth or token. [Kazuki Suda]
|
||||||
|
|
||||||
0.8.0 (2016-02-14)
|
0.8.0 (2016-02-14)
|
||||||
------------------
|
------------------
|
||||||
|
|
||||||
|
|||||||
49
README.rst
49
README.rst
@@ -4,6 +4,11 @@ github-backup
|
|||||||
|
|
||||||
backup a github user or organization
|
backup a github user or organization
|
||||||
|
|
||||||
|
Requirements
|
||||||
|
============
|
||||||
|
|
||||||
|
- GIT 1.9+
|
||||||
|
|
||||||
Installation
|
Installation
|
||||||
============
|
============
|
||||||
|
|
||||||
@@ -13,21 +18,24 @@ Using PIP via PyPI::
|
|||||||
|
|
||||||
Using PIP via Github::
|
Using PIP via Github::
|
||||||
|
|
||||||
pip install git+git://github.com/josegonzalez/python-github-backup.git#egg=github-backup
|
pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
|
||||||
|
|
||||||
Usage
|
Usage
|
||||||
=====
|
=====
|
||||||
|
|
||||||
CLI Usage is as follows::
|
CLI Usage is as follows::
|
||||||
|
|
||||||
Github Backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
|
github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
|
||||||
[-o OUTPUT_DIRECTORY] [--starred] [--watched] [--all]
|
[-o OUTPUT_DIRECTORY] [-i] [--starred] [--watched]
|
||||||
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
[--all] [--issues] [--issue-comments] [--issue-events]
|
||||||
[--pull-comments] [--pull-commits] [--labels] [--hooks]
|
[--pulls] [--pull-comments] [--pull-commits] [--labels]
|
||||||
[--milestones] [--repositories] [--wikis]
|
[--hooks] [--milestones] [--repositories] [--bare]
|
||||||
[--skip-existing] [-L [LANGUAGES [LANGUAGES ...]]]
|
[--wikis] [--skip-existing]
|
||||||
[-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
|
[-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
|
||||||
[-P] [-F] [--prefer-ssh] [-v]
|
[-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
|
||||||
|
[--prefer-ssh] [-v]
|
||||||
|
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
||||||
|
[--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
|
||||||
USER
|
USER
|
||||||
|
|
||||||
Backup a github account
|
Backup a github account
|
||||||
@@ -46,6 +54,7 @@ CLI Usage is as follows::
|
|||||||
personal access or OAuth token
|
personal access or OAuth token
|
||||||
-o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
|
-o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
|
||||||
directory at which to backup the repositories
|
directory at which to backup the repositories
|
||||||
|
-i, --incremental incremental backup
|
||||||
--starred include starred repositories in backup
|
--starred include starred repositories in backup
|
||||||
--watched include watched repositories in backup
|
--watched include watched repositories in backup
|
||||||
--all include everything in backup
|
--all include everything in backup
|
||||||
@@ -60,6 +69,7 @@ CLI Usage is as follows::
|
|||||||
authenticated)
|
authenticated)
|
||||||
--milestones include milestones in backup
|
--milestones include milestones in backup
|
||||||
--repositories include repository clone in backup
|
--repositories include repository clone in backup
|
||||||
|
--bare clone bare repositories
|
||||||
--wikis include wiki clone in backup
|
--wikis include wiki clone in backup
|
||||||
--skip-existing skip project if a backup directory exists
|
--skip-existing skip project if a backup directory exists
|
||||||
-L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
|
-L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
|
||||||
@@ -75,6 +85,12 @@ CLI Usage is as follows::
|
|||||||
-F, --fork include forked repositories
|
-F, --fork include forked repositories
|
||||||
--prefer-ssh Clone repositories using SSH instead of HTTPS
|
--prefer-ssh Clone repositories using SSH instead of HTTPS
|
||||||
-v, --version show program's version number and exit
|
-v, --version show program's version number and exit
|
||||||
|
--keychain-name OSX_KEYCHAIN_ITEM_NAME
|
||||||
|
OSX ONLY: name field of password item in OSX keychain
|
||||||
|
that holds the personal access or OAuth token
|
||||||
|
--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT
|
||||||
|
OSX ONLY: account field of password item in OSX
|
||||||
|
keychain that holds the personal access or OAuth token
|
||||||
|
|
||||||
|
|
||||||
The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
|
The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
|
||||||
@@ -83,3 +99,18 @@ Authentication
|
|||||||
==============
|
==============
|
||||||
|
|
||||||
Note: Password-based authentication will fail if you have two-factor authentication enabled.
|
Note: Password-based authentication will fail if you have two-factor authentication enabled.
|
||||||
|
|
||||||
|
Using the Keychain on Mac OSX
|
||||||
|
=============================
|
||||||
|
Note: On Mac OSX the token can be stored securely in the user's keychain. To do this:
|
||||||
|
|
||||||
|
1. Open Keychain from "Applications -> Utilities -> Keychain Access"
|
||||||
|
2. Add a new password item using "File -> New Password Item"
|
||||||
|
3. Enter a name in the "Keychain Item Name" box. You must provide this name to github-backup using the --keychain-name argument.
|
||||||
|
4. Enter an account name in the "Account Name" box, enter your Github username as set above. You must provide this name to github-backup using the --keychain-account argument.
|
||||||
|
5. Enter your Github personal access token in the "Password" box
|
||||||
|
|
||||||
|
Note: When you run github-backup, you will be asked whether you want to allow "security" to use your confidential information stored in your keychain. You have two options:
|
||||||
|
|
||||||
|
1. **Allow:** In this case you will need to click "Allow" each time you run `github-backup`
|
||||||
|
2. **Always Allow:** In this case, you will not be asked for permission when you run `github-backup` in future. This is less secure, but is required if you want to schedule `github-backup` to run automatically
|
||||||
|
|||||||
@@ -16,8 +16,23 @@ import select
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib
|
import platform
|
||||||
import urllib2
|
try:
|
||||||
|
# python 3
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from urllib.parse import quote as urlquote
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
from urllib.error import HTTPError, URLError
|
||||||
|
from urllib.request import urlopen
|
||||||
|
from urllib.request import Request
|
||||||
|
except ImportError:
|
||||||
|
# python 2
|
||||||
|
from urlparse import urlparse
|
||||||
|
from urllib import quote as urlquote
|
||||||
|
from urllib import urlencode
|
||||||
|
from urllib2 import HTTPError, URLError
|
||||||
|
from urllib2 import urlopen
|
||||||
|
from urllib2 import Request
|
||||||
|
|
||||||
from github_backup import __version__
|
from github_backup import __version__
|
||||||
|
|
||||||
@@ -79,8 +94,8 @@ def logging_subprocess(popenargs,
|
|||||||
rc = child.wait()
|
rc = child.wait()
|
||||||
|
|
||||||
if rc != 0:
|
if rc != 0:
|
||||||
print(u'{} returned {}:'.format(popenargs[0], rc), file=sys.stderr)
|
print('{} returned {}:'.format(popenargs[0], rc), file=sys.stderr)
|
||||||
print('\t', u' '.join(popenargs), file=sys.stderr)
|
print('\t', ' '.join(popenargs), file=sys.stderr)
|
||||||
|
|
||||||
return rc
|
return rc
|
||||||
|
|
||||||
@@ -96,9 +111,19 @@ def mkdir_p(*args):
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def mask_password(url, secret='*****'):
|
||||||
|
parsed = urlparse(url)
|
||||||
|
|
||||||
|
if not parsed.password:
|
||||||
|
return url
|
||||||
|
elif parsed.password == 'x-oauth-basic':
|
||||||
|
return url.replace(parsed.username, secret)
|
||||||
|
|
||||||
|
return url.replace(parsed.password, secret)
|
||||||
|
|
||||||
|
|
||||||
def parse_args():
|
def parse_args():
|
||||||
parser = argparse.ArgumentParser(description='Backup a github account',
|
parser = argparse.ArgumentParser(description='Backup a github account')
|
||||||
prog='Github Backup')
|
|
||||||
parser.add_argument('user',
|
parser.add_argument('user',
|
||||||
metavar='USER',
|
metavar='USER',
|
||||||
type=str,
|
type=str,
|
||||||
@@ -116,12 +141,17 @@ def parse_args():
|
|||||||
parser.add_argument('-t',
|
parser.add_argument('-t',
|
||||||
'--token',
|
'--token',
|
||||||
dest='token',
|
dest='token',
|
||||||
help='personal access or OAuth token')
|
help='personal access or OAuth token, or path to token (file://...)') # noqa
|
||||||
parser.add_argument('-o',
|
parser.add_argument('-o',
|
||||||
'--output-directory',
|
'--output-directory',
|
||||||
default='.',
|
default='.',
|
||||||
dest='output_directory',
|
dest='output_directory',
|
||||||
help='directory at which to backup the repositories')
|
help='directory at which to backup the repositories')
|
||||||
|
parser.add_argument('-i',
|
||||||
|
'--incremental',
|
||||||
|
action='store_true',
|
||||||
|
dest='incremental',
|
||||||
|
help='incremental backup')
|
||||||
parser.add_argument('--starred',
|
parser.add_argument('--starred',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_starred',
|
dest='include_starred',
|
||||||
@@ -165,7 +195,7 @@ def parse_args():
|
|||||||
parser.add_argument('--hooks',
|
parser.add_argument('--hooks',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_hooks',
|
dest='include_hooks',
|
||||||
help='include hooks in backup (works only when authenticated)')
|
help='include hooks in backup (works only when authenticated)') # noqa
|
||||||
parser.add_argument('--milestones',
|
parser.add_argument('--milestones',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_milestones',
|
dest='include_milestones',
|
||||||
@@ -174,6 +204,10 @@ def parse_args():
|
|||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_repository',
|
dest='include_repository',
|
||||||
help='include repository clone in backup')
|
help='include repository clone in backup')
|
||||||
|
parser.add_argument('--bare',
|
||||||
|
action='store_true',
|
||||||
|
dest='bare_clone',
|
||||||
|
help='clone bare repositories')
|
||||||
parser.add_argument('--wikis',
|
parser.add_argument('--wikis',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='include_wiki',
|
dest='include_wiki',
|
||||||
@@ -218,23 +252,61 @@ def parse_args():
|
|||||||
parser.add_argument('-v', '--version',
|
parser.add_argument('-v', '--version',
|
||||||
action='version',
|
action='version',
|
||||||
version='%(prog)s ' + __version__)
|
version='%(prog)s ' + __version__)
|
||||||
|
parser.add_argument('--keychain-name',
|
||||||
|
dest='osx_keychain_item_name',
|
||||||
|
help='OSX ONLY: name field of password item in OSX keychain that holds the personal access or OAuth token')
|
||||||
|
parser.add_argument('--keychain-account',
|
||||||
|
dest='osx_keychain_item_account',
|
||||||
|
help='OSX ONLY: account field of password item in OSX keychain that holds the personal access or OAuth token')
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
def get_auth(args):
|
def get_auth(args, encode=True):
|
||||||
if args.token:
|
auth = None
|
||||||
return base64.b64encode(args.token + ':' + 'x-oauth-basic')
|
|
||||||
|
|
||||||
if args.username:
|
if args.osx_keychain_item_name:
|
||||||
|
if not args.osx_keychain_item_account:
|
||||||
|
log_error('You must specify both name and account fields for osx keychain password items')
|
||||||
|
else:
|
||||||
|
if platform.system() != 'Darwin':
|
||||||
|
log_error("Keychain arguments are only supported on Mac OSX")
|
||||||
|
try:
|
||||||
|
with open(os.devnull,'w') as devnull:
|
||||||
|
token = (subprocess.check_output([
|
||||||
|
'security','find-generic-password',
|
||||||
|
'-s',args.osx_keychain_item_name,
|
||||||
|
'-a',args.osx_keychain_item_account,
|
||||||
|
'-w' ], stderr=devnull).strip())
|
||||||
|
auth = token + ':' + 'x-oauth-basic'
|
||||||
|
except:
|
||||||
|
log_error('No password item matching the provided name and account could be found in the osx keychain.')
|
||||||
|
elif args.osx_keychain_item_account:
|
||||||
|
log_error('You must specify both name and account fields for osx keychain password items')
|
||||||
|
elif args.token:
|
||||||
|
_path_specifier = 'file://'
|
||||||
|
if args.token.startswith(_path_specifier):
|
||||||
|
args.token = open(args.token[len(_path_specifier):],
|
||||||
|
'rt').readline().strip()
|
||||||
|
auth = args.token + ':' + 'x-oauth-basic'
|
||||||
|
elif args.username:
|
||||||
if not args.password:
|
if not args.password:
|
||||||
args.password = getpass.getpass()
|
args.password = getpass.getpass()
|
||||||
return base64.b64encode(args.username + ':' + args.password)
|
if encode:
|
||||||
|
password = args.password
|
||||||
if args.password:
|
else:
|
||||||
|
password = urlquote(args.password)
|
||||||
|
auth = args.username + ':' + password
|
||||||
|
elif args.password:
|
||||||
log_error('You must specify a username for basic auth')
|
log_error('You must specify a username for basic auth')
|
||||||
|
|
||||||
|
if not auth:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
if not encode:
|
||||||
|
return auth
|
||||||
|
|
||||||
|
return base64.b64encode(auth.encode('ascii'))
|
||||||
|
|
||||||
|
|
||||||
def get_github_api_host(args):
|
def get_github_api_host(args):
|
||||||
if args.github_host:
|
if args.github_host:
|
||||||
@@ -245,7 +317,7 @@ def get_github_api_host(args):
|
|||||||
return host
|
return host
|
||||||
|
|
||||||
|
|
||||||
def get_github_ssh_host(args):
|
def get_github_host(args):
|
||||||
if args.github_host:
|
if args.github_host:
|
||||||
host = args.github_host
|
host = args.github_host
|
||||||
else:
|
else:
|
||||||
@@ -254,6 +326,23 @@ def get_github_ssh_host(args):
|
|||||||
return host
|
return host
|
||||||
|
|
||||||
|
|
||||||
|
def get_github_repo_url(args, repository):
|
||||||
|
if args.prefer_ssh:
|
||||||
|
return repository['ssh_url']
|
||||||
|
|
||||||
|
auth = get_auth(args, False)
|
||||||
|
if auth:
|
||||||
|
repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
|
||||||
|
auth,
|
||||||
|
get_github_host(args),
|
||||||
|
args.user,
|
||||||
|
repository['name'])
|
||||||
|
else:
|
||||||
|
repo_url = repository['clone_url']
|
||||||
|
|
||||||
|
return repo_url
|
||||||
|
|
||||||
|
|
||||||
def retrieve_data(args, template, query_args=None, single_request=False):
|
def retrieve_data(args, template, query_args=None, single_request=False):
|
||||||
auth = get_auth(args)
|
auth = get_auth(args)
|
||||||
query_args = get_query_args(query_args)
|
query_args = get_query_args(query_args)
|
||||||
@@ -273,7 +362,7 @@ def retrieve_data(args, template, query_args=None, single_request=False):
|
|||||||
errors.append(template.format(status_code, r.reason))
|
errors.append(template.format(status_code, r.reason))
|
||||||
log_error(errors)
|
log_error(errors)
|
||||||
|
|
||||||
response = json.loads(r.read())
|
response = json.loads(r.read().decode('utf-8'))
|
||||||
if len(errors) == 0:
|
if len(errors) == 0:
|
||||||
if type(response) == list:
|
if type(response) == list:
|
||||||
data.extend(response)
|
data.extend(response)
|
||||||
@@ -305,11 +394,11 @@ def _get_response(request, auth, template):
|
|||||||
while True:
|
while True:
|
||||||
should_continue = False
|
should_continue = False
|
||||||
try:
|
try:
|
||||||
r = urllib2.urlopen(request)
|
r = urlopen(request)
|
||||||
except urllib2.HTTPError as exc:
|
except HTTPError as exc:
|
||||||
errors, should_continue = _request_http_error(exc, auth, errors) # noqa
|
errors, should_continue = _request_http_error(exc, auth, errors) # noqa
|
||||||
r = exc
|
r = exc
|
||||||
except urllib2.URLError:
|
except URLError:
|
||||||
should_continue = _request_url_error(template, retry_timeout)
|
should_continue = _request_url_error(template, retry_timeout)
|
||||||
if not should_continue:
|
if not should_continue:
|
||||||
raise
|
raise
|
||||||
@@ -322,14 +411,14 @@ def _get_response(request, auth, template):
|
|||||||
|
|
||||||
|
|
||||||
def _construct_request(per_page, page, query_args, template, auth):
|
def _construct_request(per_page, page, query_args, template, auth):
|
||||||
querystring = urllib.urlencode(dict({
|
querystring = urlencode(dict(list({
|
||||||
'per_page': per_page,
|
'per_page': per_page,
|
||||||
'page': page
|
'page': page
|
||||||
}.items() + query_args.items()))
|
}.items()) + list(query_args.items())))
|
||||||
|
|
||||||
request = urllib2.Request(template + '?' + querystring)
|
request = Request(template + '?' + querystring)
|
||||||
if auth is not None:
|
if auth is not None:
|
||||||
request.add_header('Authorization', 'Basic ' + auth)
|
request.add_header('Authorization', 'Basic '.encode('ascii') + auth)
|
||||||
return request
|
return request
|
||||||
|
|
||||||
|
|
||||||
@@ -356,10 +445,9 @@ def _request_http_error(exc, auth, errors):
|
|||||||
print('Exceeded rate limit of {} requests; waiting {} seconds to reset'.format(limit, delta), # noqa
|
print('Exceeded rate limit of {} requests; waiting {} seconds to reset'.format(limit, delta), # noqa
|
||||||
file=sys.stderr)
|
file=sys.stderr)
|
||||||
|
|
||||||
ratelimit_error = 'No more requests remaining'
|
|
||||||
if auth is None:
|
if auth is None:
|
||||||
ratelimit_error += '; authenticate to raise your GitHub rate limit' # noqa
|
print('Hint: Authenticate to raise your GitHub rate limit',
|
||||||
errors.append(ratelimit_error)
|
file=sys.stderr)
|
||||||
|
|
||||||
time.sleep(delta)
|
time.sleep(delta)
|
||||||
should_continue = True
|
should_continue = True
|
||||||
@@ -399,10 +487,13 @@ def retrieve_repositories(args):
|
|||||||
return retrieve_data(args, template, single_request=single_request)
|
return retrieve_data(args, template, single_request=single_request)
|
||||||
|
|
||||||
|
|
||||||
def filter_repositories(args, repositories):
|
def filter_repositories(args, unfiltered_repositories):
|
||||||
log_info('Filtering repositories')
|
log_info('Filtering repositories')
|
||||||
|
|
||||||
repositories = [r for r in repositories if r['owner']['login'] == args.user]
|
repositories = []
|
||||||
|
for r in unfiltered_repositories:
|
||||||
|
if r['owner']['login'] == args.user:
|
||||||
|
repositories.append(r)
|
||||||
|
|
||||||
name_regex = None
|
name_regex = None
|
||||||
if args.name_regex:
|
if args.name_regex:
|
||||||
@@ -428,28 +519,36 @@ def backup_repositories(args, output_directory, repositories):
|
|||||||
log_info('Backing up repositories')
|
log_info('Backing up repositories')
|
||||||
repos_template = 'https://{0}/repos'.format(get_github_api_host(args))
|
repos_template = 'https://{0}/repos'.format(get_github_api_host(args))
|
||||||
|
|
||||||
|
if args.incremental:
|
||||||
|
last_update = max(repository['updated_at'] for repository in repositories) # noqa
|
||||||
|
last_update_path = os.path.join(output_directory, 'last_update')
|
||||||
|
if os.path.exists(last_update_path):
|
||||||
|
args.since = open(last_update_path).read().strip()
|
||||||
|
else:
|
||||||
|
args.since = None
|
||||||
|
else:
|
||||||
|
args.since = None
|
||||||
|
|
||||||
for repository in repositories:
|
for repository in repositories:
|
||||||
backup_cwd = os.path.join(output_directory, 'repositories')
|
backup_cwd = os.path.join(output_directory, 'repositories')
|
||||||
repo_cwd = os.path.join(backup_cwd, repository['name'])
|
repo_cwd = os.path.join(backup_cwd, repository['name'])
|
||||||
repo_dir = os.path.join(repo_cwd, 'repository')
|
repo_dir = os.path.join(repo_cwd, 'repository')
|
||||||
|
repo_url = get_github_repo_url(args, repository)
|
||||||
if args.prefer_ssh:
|
|
||||||
repo_url = repository['ssh_url']
|
|
||||||
else:
|
|
||||||
repo_url = repository['clone_url']
|
|
||||||
|
|
||||||
if args.include_repository or args.include_everything:
|
if args.include_repository or args.include_everything:
|
||||||
fetch_repository(repository['name'],
|
fetch_repository(repository['name'],
|
||||||
repo_url,
|
repo_url,
|
||||||
repo_dir,
|
repo_dir,
|
||||||
skip_existing=args.skip_existing)
|
skip_existing=args.skip_existing,
|
||||||
|
bare_clone=args.bare_clone)
|
||||||
|
|
||||||
download_wiki = (args.include_wiki or args.include_everything)
|
download_wiki = (args.include_wiki or args.include_everything)
|
||||||
if repository['has_wiki'] and download_wiki:
|
if repository['has_wiki'] and download_wiki:
|
||||||
fetch_repository(repository['name'],
|
fetch_repository(repository['name'],
|
||||||
repo_url.replace('.git', '.wiki.git'),
|
repo_url.replace('.git', '.wiki.git'),
|
||||||
os.path.join(repo_cwd, 'wiki'),
|
os.path.join(repo_cwd, 'wiki'),
|
||||||
skip_existing=args.skip_existing)
|
skip_existing=args.skip_existing,
|
||||||
|
bare_clone=args.bare_clone)
|
||||||
|
|
||||||
if args.include_issues or args.include_everything:
|
if args.include_issues or args.include_everything:
|
||||||
backup_issues(args, repo_cwd, repository, repos_template)
|
backup_issues(args, repo_cwd, repository, repos_template)
|
||||||
@@ -466,6 +565,9 @@ def backup_repositories(args, output_directory, repositories):
|
|||||||
if args.include_hooks or args.include_everything:
|
if args.include_hooks or args.include_everything:
|
||||||
backup_hooks(args, repo_cwd, repository, repos_template)
|
backup_hooks(args, repo_cwd, repository, repos_template)
|
||||||
|
|
||||||
|
if args.incremental:
|
||||||
|
open(last_update_path, 'w').write(last_update)
|
||||||
|
|
||||||
|
|
||||||
def backup_issues(args, repo_cwd, repository, repos_template):
|
def backup_issues(args, repo_cwd, repository, repos_template):
|
||||||
has_issues_dir = os.path.isdir('{0}/issues/.git'.format(repo_cwd))
|
has_issues_dir = os.path.isdir('{0}/issues/.git'.format(repo_cwd))
|
||||||
@@ -482,12 +584,15 @@ def backup_issues(args, repo_cwd, repository, repos_template):
|
|||||||
_issue_template = '{0}/{1}/issues'.format(repos_template,
|
_issue_template = '{0}/{1}/issues'.format(repos_template,
|
||||||
repository['full_name'])
|
repository['full_name'])
|
||||||
|
|
||||||
|
should_include_pulls = args.include_pulls or args.include_everything
|
||||||
issue_states = ['open', 'closed']
|
issue_states = ['open', 'closed']
|
||||||
for issue_state in issue_states:
|
for issue_state in issue_states:
|
||||||
query_args = {
|
query_args = {
|
||||||
'filter': 'all',
|
'filter': 'all',
|
||||||
'state': issue_state
|
'state': issue_state
|
||||||
}
|
}
|
||||||
|
if args.since:
|
||||||
|
query_args['since'] = args.since
|
||||||
|
|
||||||
_issues = retrieve_data(args,
|
_issues = retrieve_data(args,
|
||||||
_issue_template,
|
_issue_template,
|
||||||
@@ -495,18 +600,21 @@ def backup_issues(args, repo_cwd, repository, repos_template):
|
|||||||
for issue in _issues:
|
for issue in _issues:
|
||||||
# skip pull requests which are also returned as issues
|
# skip pull requests which are also returned as issues
|
||||||
# if retrieving pull requests is requested as well
|
# if retrieving pull requests is requested as well
|
||||||
if 'pull_request' in issue and (args.include_pulls or args.include_everything):
|
if 'pull_request' in issue and should_include_pulls:
|
||||||
issues_skipped += 1
|
issues_skipped += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
issues[issue['number']] = issue
|
issues[issue['number']] = issue
|
||||||
|
|
||||||
if issues_skipped:
|
if issues_skipped:
|
||||||
issues_skipped_message = ' (skipped {0} pull requests)'.format(issues_skipped)
|
issues_skipped_message = ' (skipped {0} pull requests)'.format(
|
||||||
log_info('Saving {0} issues to disk{1}'.format(len(issues.keys()), issues_skipped_message))
|
issues_skipped)
|
||||||
|
|
||||||
|
log_info('Saving {0} issues to disk{1}'.format(
|
||||||
|
len(list(issues.keys())), issues_skipped_message))
|
||||||
comments_template = _issue_template + '/{0}/comments'
|
comments_template = _issue_template + '/{0}/comments'
|
||||||
events_template = _issue_template + '/{0}/events'
|
events_template = _issue_template + '/{0}/events'
|
||||||
for number, issue in issues.iteritems():
|
for number, issue in list(issues.items()):
|
||||||
if args.include_issue_comments or args.include_everything:
|
if args.include_issue_comments or args.include_everything:
|
||||||
template = comments_template.format(number)
|
template = comments_template.format(number)
|
||||||
issues[number]['comment_data'] = retrieve_data(args, template)
|
issues[number]['comment_data'] = retrieve_data(args, template)
|
||||||
@@ -536,19 +644,24 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
|
|||||||
for pull_state in pull_states:
|
for pull_state in pull_states:
|
||||||
query_args = {
|
query_args = {
|
||||||
'filter': 'all',
|
'filter': 'all',
|
||||||
'state': pull_state
|
'state': pull_state,
|
||||||
|
'sort': 'updated',
|
||||||
|
'direction': 'desc',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# It'd be nice to be able to apply the args.since filter here...
|
||||||
_pulls = retrieve_data(args,
|
_pulls = retrieve_data(args,
|
||||||
_pulls_template,
|
_pulls_template,
|
||||||
query_args=query_args)
|
query_args=query_args)
|
||||||
for pull in _pulls:
|
for pull in _pulls:
|
||||||
|
if not args.since or pull['updated_at'] >= args.since:
|
||||||
pulls[pull['number']] = pull
|
pulls[pull['number']] = pull
|
||||||
|
|
||||||
log_info('Saving {0} pull requests to disk'.format(len(pulls.keys())))
|
log_info('Saving {0} pull requests to disk'.format(
|
||||||
|
len(list(pulls.keys()))))
|
||||||
comments_template = _pulls_template + '/{0}/comments'
|
comments_template = _pulls_template + '/{0}/comments'
|
||||||
commits_template = _pulls_template + '/{0}/commits'
|
commits_template = _pulls_template + '/{0}/commits'
|
||||||
for number, pull in pulls.iteritems():
|
for number, pull in list(pulls.items()):
|
||||||
if args.include_pull_comments or args.include_everything:
|
if args.include_pull_comments or args.include_everything:
|
||||||
template = comments_template.format(number)
|
template = comments_template.format(number)
|
||||||
pulls[number]['comment_data'] = retrieve_data(args, template)
|
pulls[number]['comment_data'] = retrieve_data(args, template)
|
||||||
@@ -582,8 +695,9 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
|
|||||||
for milestone in _milestones:
|
for milestone in _milestones:
|
||||||
milestones[milestone['number']] = milestone
|
milestones[milestone['number']] = milestone
|
||||||
|
|
||||||
log_info('Saving {0} milestones to disk'.format(len(milestones.keys())))
|
log_info('Saving {0} milestones to disk'.format(
|
||||||
for number, milestone in milestones.iteritems():
|
len(list(milestones.keys()))))
|
||||||
|
for number, milestone in list(milestones.items()):
|
||||||
milestone_file = '{0}/{1}.json'.format(milestone_cwd, number)
|
milestone_file = '{0}/{1}.json'.format(milestone_cwd, number)
|
||||||
with codecs.open(milestone_file, 'w', encoding='utf-8') as f:
|
with codecs.open(milestone_file, 'w', encoding='utf-8') as f:
|
||||||
json_dump(milestone, f)
|
json_dump(milestone, f)
|
||||||
@@ -620,32 +734,62 @@ def backup_hooks(args, repo_cwd, repository, repos_template):
|
|||||||
log_info("Unable to read hooks, skipping")
|
log_info("Unable to read hooks, skipping")
|
||||||
|
|
||||||
|
|
||||||
def fetch_repository(name, remote_url, local_dir, skip_existing=False):
|
def fetch_repository(name,
|
||||||
|
remote_url,
|
||||||
|
local_dir,
|
||||||
|
skip_existing=False,
|
||||||
|
bare_clone=False):
|
||||||
|
if bare_clone:
|
||||||
|
if os.path.exists(local_dir):
|
||||||
|
clone_exists = subprocess.check_output(['git',
|
||||||
|
'rev-parse',
|
||||||
|
'--is-bare-repository'],
|
||||||
|
cwd=local_dir) == "true\n"
|
||||||
|
else:
|
||||||
|
clone_exists = False
|
||||||
|
else:
|
||||||
clone_exists = os.path.exists(os.path.join(local_dir, '.git'))
|
clone_exists = os.path.exists(os.path.join(local_dir, '.git'))
|
||||||
|
|
||||||
if clone_exists and skip_existing:
|
if clone_exists and skip_existing:
|
||||||
return
|
return
|
||||||
|
|
||||||
initalized = subprocess.call('git ls-remote ' + remote_url,
|
masked_remote_url = mask_password(remote_url)
|
||||||
|
|
||||||
|
initialized = subprocess.call('git ls-remote ' + remote_url,
|
||||||
stdout=FNULL,
|
stdout=FNULL,
|
||||||
stderr=FNULL,
|
stderr=FNULL,
|
||||||
shell=True)
|
shell=True)
|
||||||
if initalized == 128:
|
if initialized == 128:
|
||||||
log_info("Skipping {0} ({1}) since it's not initalized".format(name, remote_url))
|
log_info("Skipping {0} ({1}) since it's not initialized".format(
|
||||||
|
name, masked_remote_url))
|
||||||
return
|
return
|
||||||
|
|
||||||
if clone_exists:
|
if clone_exists:
|
||||||
log_info('Updating {0} in {1}'.format(name, local_dir))
|
log_info('Updating {0} in {1}'.format(name, local_dir))
|
||||||
|
|
||||||
|
remotes = subprocess.check_output(['git', 'remote', 'show'],
|
||||||
|
cwd=local_dir)
|
||||||
|
remotes = [i.strip() for i in remotes.decode('utf-8')]
|
||||||
|
|
||||||
|
if 'origin' not in remotes:
|
||||||
git_command = ['git', 'remote', 'rm', 'origin']
|
git_command = ['git', 'remote', 'rm', 'origin']
|
||||||
logging_subprocess(git_command, None, cwd=local_dir)
|
logging_subprocess(git_command, None, cwd=local_dir)
|
||||||
git_command = ['git', 'remote', 'add', 'origin', remote_url]
|
git_command = ['git', 'remote', 'add', 'origin', remote_url]
|
||||||
logging_subprocess(git_command, None, cwd=local_dir)
|
logging_subprocess(git_command, None, cwd=local_dir)
|
||||||
git_command = ['git', 'fetch', '--all', '--tags', '--prune']
|
else:
|
||||||
|
git_command = ['git', 'remote', 'set-url', 'origin', remote_url]
|
||||||
|
logging_subprocess(git_command, None, cwd=local_dir)
|
||||||
|
|
||||||
|
git_command = ['git', 'fetch', '--all', '--force', '--tags', '--prune']
|
||||||
logging_subprocess(git_command, None, cwd=local_dir)
|
logging_subprocess(git_command, None, cwd=local_dir)
|
||||||
else:
|
else:
|
||||||
log_info('Cloning {0} repository from {1} to {2}'.format(name,
|
log_info('Cloning {0} repository from {1} to {2}'.format(
|
||||||
remote_url,
|
name,
|
||||||
|
masked_remote_url,
|
||||||
local_dir))
|
local_dir))
|
||||||
|
if bare_clone:
|
||||||
|
git_command = ['git', 'clone', '--mirror', remote_url, local_dir]
|
||||||
|
else:
|
||||||
git_command = ['git', 'clone', remote_url, local_dir]
|
git_command = ['git', 'clone', remote_url, local_dir]
|
||||||
logging_subprocess(git_command, None)
|
logging_subprocess(git_command, None)
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
__version__ = '0.8.0'
|
__version__ = '0.13.0'
|
||||||
|
|||||||
2
release
2
release
@@ -34,7 +34,7 @@ fi
|
|||||||
echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
|
echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
|
||||||
|
|
||||||
set +e;
|
set +e;
|
||||||
git status | grep "working directory clean" &> /dev/null
|
git status | grep -Eo "working (directory|tree) clean" &> /dev/null
|
||||||
if [ ! $? -eq 0 ]; then # working directory is NOT clean
|
if [ ! $? -eq 0 ]; then # working directory is NOT clean
|
||||||
echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
|
echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
Reference in New Issue
Block a user