diff --git a/utils.py b/utils.py index 5601845..3a781ba 100644 --- a/utils.py +++ b/utils.py @@ -1,20 +1,36 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse +import getpass import os +import shutil import sys import urllib.error import urllib.request def downloadModel(destination): modelUrl = "https://github.com/coosto/dutch-word-embeddings/releases/download/v1.0/model.bin" - sys.stderr.write("Downloading model... ({})\n".format(modelUrl)) + + # Optional token to speed up the download + username = getpass.getpass("(Optional, but increases download speed)\nGitHub username:") + token = getpass.getpass("GitHub Token:") + if username and token: + password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() + password_mgr.add_password(None, "https://github.com/coosto/", username, token) + handler = urllib.request.HTTPBasicAuthHandler(password_mgr) + opener = urllib.request.build_opener(handler) + urllib.request.install_opener(opener) + + # Request model data try: - outputFile, result = urllib.request.urlretrieve(modelUrl, destination) + request = urllib.request.Request(modelUrl) + with urllib.request.urlopen(request) as response, open(destination, 'wb') as fp: + shutil.copyfileobj(response, fp) + sys.stderr.write("Model saved: {}\n".format(fp.name)) except urllib.error.HTTPError as error: sys.stderr.write("Error fetching model from github.com - {}\n".format(error)) + sys.stderr.write("{}".format(error.read())) exit(1) - sys.stderr.write("Model saved: {}\n".format(outputFile)) def demoAnalogies(model):