Add OAuth token support, docs and better arg parsing via docopt

Author: HR

Pipfile

@@ -6,4 +6,5 @@ name = "pypi"
 [dev-packages]
 
 [packages]
-requests = ">=2.20.0"
+requests = ">=2.20.0"
+docopt = "*"

Pipfile.lock

@@ -1,2 +1,35 @@
-# github-clone
-Recursively clone a GitHub repo sub-dir
+# GitHub clone
+Git clone sub-directories of a GitHub repository (at any reference) without having to clone the entire repository.
+Uses the GitHub API to recursively clone the sub-directories tree and files.
+
+# Rate limit
+The GitHub API imposes a [rate limiting](https://developer.github.com/v3/#rate-limiting) of up to 60 requests per hour applies but can be increased to up to 5000 requests per hour using an _OAuth token_ (to get one see https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line).
+
+# Private repositories
+To clone private repositories you need to supply an _OAuth token_ for an account with access to the private repository (to get one see https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line).
+
+# Installation
+
+
+# Usage
+```
+GitHub clone (git.io/ghclone)
+
+Usage:
+  ghclone.py <url> [-t | --token=<token>]
+  ghclone.py (-h | --help)
+  ghclone.py (-v | --version)
+
+Examples:
+  ghclone.py https://github.com/HR/Crypter/tree/master/app
+  ghclone.py https://github.com/HR/Crypter/tree/dev/app
+  ghclone.py https://github.com/HR/Crypter/tree/v3.1.0/build
+  ghclone.py https://github.com/HR/Crypter/tree/cbee54dd720bb8aaa3a2111fcec667ca5f700510/build
+  ghclone.py https://github.com/HR/Picturesque/tree/master/app/src -t li50d67757gm20556d53f08126215725a698560b
+
+Options:
+  -h --help           Show this screen.
+  -v --version        Show version.
+  -t --token=<token>  Set a GitHub OAuth token (see https://developer.github.com/v3/#rate-limiting).
+```
+

README.md

@@ -1,14 +1,40 @@
+"""
+GitHub clone (git.io/ghclone)
+
+Usage:
+  ghclone.py <url> [-t | --token=<token>]
+  ghclone.py (-h | --help)
+  ghclone.py (-v | --version)
+
+Examples:
+  ghclone.py https://github.com/HR/Crypter/tree/master/app
+  ghclone.py https://github.com/HR/Crypter/tree/dev/app
+  ghclone.py https://github.com/HR/Crypter/tree/v3.1.0/build
+  ghclone.py https://github.com/HR/Crypter/tree/cbee54dd720bb8aaa3a2111fcec667ca5f700510/build
+  ghclone.py https://github.com/HR/Picturesque/tree/master/app/src -t li50d67757gm20556d53f08126215725a698560b
+
+Options:
+  -h --help           Show this screen.
+  -v --version        Show version.
+  -t --token=<token>  Set a GitHub OAuth token (see https://developer.github.com/v3/#rate-limiting).
+
+(C) 2019 Habib Rehman (git.io/HR)
+"""
 import requests
 import re
 import sys
 import os
 import errno
+from docopt import docopt
 
-
+VERSION = '1.0.0'
 GH_API_BASE_URL = 'https://api.github.com'
 GH_REPO_CONTENTS_ENDPOINT = GH_API_BASE_URL + '/repos/{}/{}/contents'
 BASE_NORMALIZE_REGEX = re.compile(r'.*github\.com\/')
 
+verbose = False
+req = requests.Session()
+req.headers.update({'User-Agent': 'git.io/ghclone '+VERSION})
 
 def exit_with_m(m='An error occured'):
     print(m)
@@ -27,11 +53,11 @@ def clone_file(download_url, file_path):
     """
     Clones the file at the download_url to the file_path
     """
-    r = requests.get(download_url, stream=True)
+    r = req.get(download_url, stream=True)
     try:
         r.raise_for_status()
     except Exception as e:
-        exit_with_m('Failed cloneing ' + download_url, e)
+        exit_with_m('Failed cloning ' + download_url, e)
 
     with open(file_path, 'wb') as fd:
         for chunk in r.iter_content(chunk_size=128):
@@ -43,11 +69,11 @@ def clone(base_url, path=None, ref=None):
     """
     req_url = base_url if not path else os.path.join(base_url, path)
     # Get path metadata
-    r = requests.get(req_url) if not ref else requests.get(req_url, params={'ref': ref})
+    r = req.get(req_url) if not ref else req.get(req_url, params={'ref': ref})
     try:
         r.raise_for_status()
     except Exception as e:
-        exit_with_m('Failed fetching metadata of dir: ', e)
+        exit_with_m('Failed fetching metadata for ' + path, e)
     repo_data = r.json()
 
     # Create path locally
@@ -68,20 +94,24 @@ def clone(base_url, path=None, ref=None):
 ###
 # Main
 ###
-arg_len = len(sys.argv)
-if arg_len >= 2:
-    # Github URL
-    gh_url = sys.argv[1]
+if __name__ == '__main__':
+    arguments = docopt(__doc__)
+    if arguments['--version']:
+        print(VERSION)
+        sys.exit(0)
+
+    # Get params
+    gh_url = arguments['<url>']
+    token = arguments['--token']
+    if token:
+        req.headers.update({'Authorization': 'token '+token[0]})
     # Normalize & parse input
     normal_gh_url = re.sub(BASE_NORMALIZE_REGEX, '', gh_url).replace('/tree', '')
     gh_url_comps = normal_gh_url.split('/')
     user, repo = gh_url_comps[:2]
     ref = gh_url_comps[2]
     path = os.path.join(*gh_url_comps[3:])
-else:
-    exit_with_m('Nothing to clone :(')
-
-api_req_url = GH_REPO_CONTENTS_ENDPOINT.format(user, repo)
-print("Cloning into '%s'..." % path)
-clone(api_req_url, path, ref)
-print("done.")
+    api_req_url = GH_REPO_CONTENTS_ENDPOINT.format(user, repo)
+    print("Cloning into '%s'..." % path)
+    clone(api_req_url, path, ref)
+    print("done.")