diff --git a/ah_bootstrap.py b/ah_bootstrap.py index 0dc50071b..786b8b14d 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -91,13 +91,24 @@ use_setuptools() +# typing as a dependency for 1.6.1+ Sphinx causes issues when imported after +# initializing submodule with ah_boostrap.py +# See discussion and references in +# https://github.com/astropy/astropy-helpers/issues/302 + +try: + import typing # noqa +except ImportError: + pass + + # Note: The following import is required as a workaround to # https://github.com/astropy/astropy-helpers/issues/89; if we don't import this # module now, it will get cleaned up after `run_setup` is called, but that will # later cause the TemporaryDirectory class defined in it to stop working when # used later on by setuptools try: - import setuptools.py31compat + import setuptools.py31compat # noqa except ImportError: pass @@ -702,7 +713,7 @@ def _update_submodule(self, submodule, status): if self.offline: cmd.append('--no-fetch') elif status == 'U': - raise _AHBoostrapSystemExit( + raise _AHBootstrapSystemExit( 'Error: Submodule {0} contains unresolved merge conflicts. ' 'Please complete or abandon any changes in the submodule so that ' 'it is in a usable state, then try again.'.format(submodule)) @@ -763,7 +774,7 @@ def run_cmd(cmd): msg = 'Command not found: `{0}`'.format(' '.join(cmd)) raise _CommandNotFound(msg, cmd) else: - raise _AHBoostrapSystemExit( + raise _AHBootstrapSystemExit( 'An unexpected error occurred when running the ' '`{0}` command:\n{1}'.format(' '.join(cmd), str(e))) @@ -878,46 +889,6 @@ def __init__(self, *args): super(_AHBootstrapSystemExit, self).__init__(msg, *args[1:]) -if sys.version_info[:2] < (2, 7): - # In Python 2.6 the distutils log does not log warnings, errors, etc. to - # stderr so we have to wrap it to ensure consistency at least in this - # module - import distutils - - class log(object): - def __getattr__(self, attr): - return getattr(distutils.log, attr) - - def warn(self, msg, *args): - self._log_to_stderr(distutils.log.WARN, msg, *args) - - def error(self, msg): - self._log_to_stderr(distutils.log.ERROR, msg, *args) - - def fatal(self, msg): - self._log_to_stderr(distutils.log.FATAL, msg, *args) - - def log(self, level, msg, *args): - if level in (distutils.log.WARN, distutils.log.ERROR, - distutils.log.FATAL): - self._log_to_stderr(level, msg, *args) - else: - distutils.log.log(level, msg, *args) - - def _log_to_stderr(self, level, msg, *args): - # This is the only truly 'public' way to get the current threshold - # of the log - current_threshold = distutils.log.set_threshold(distutils.log.WARN) - distutils.log.set_threshold(current_threshold) - if level >= current_threshold: - if args: - msg = msg % args - sys.stderr.write('%s\n' % msg) - sys.stderr.flush() - - log = log() - - BOOTSTRAPPER = _Bootstrapper.main() diff --git a/astropy_helpers b/astropy_helpers index 70b09af7e..ed2e78978 160000 --- a/astropy_helpers +++ b/astropy_helpers @@ -1 +1 @@ -Subproject commit 70b09af7e4f6b8a4e409412a913ff2adb05c12b7 +Subproject commit ed2e7897862ae9e979b336df670d7a5541cdd8f0 diff --git a/ez_setup.py b/ez_setup.py index 0842c6e43..800c31ef6 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,70 +1,62 @@ -#!python -"""Bootstrap setuptools installation +#!/usr/bin/env python -If you want to use setuptools in your package's setup.py, just include this -file in the same directory with it, and add this to the top of your setup.py:: +""" +Setuptools bootstrapping installer. - from ez_setup import use_setuptools - use_setuptools() +Maintained at https://github.com/pypa/setuptools/tree/bootstrap. -If you want to require a specific version of setuptools, set a download -mirror, or use an alternate download directory, you can do so by supplying -the appropriate options to ``use_setuptools()``. +Run this script to install or upgrade setuptools. -This file can also be run as a script to install or upgrade setuptools. +This method is DEPRECATED. Check https://github.com/pypa/setuptools/issues/581 for more details. """ + import os import shutil import sys import tempfile -import tarfile +import zipfile import optparse import subprocess import platform +import textwrap +import contextlib from distutils import log +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen + try: from site import USER_SITE except ImportError: USER_SITE = None -DEFAULT_VERSION = "1.4.2" -DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" +# 33.1.1 is the last version that supports setuptools self upgrade/installation. +DEFAULT_VERSION = "33.1.1" +DEFAULT_URL = "https://pypi.io/packages/source/s/setuptools/" +DEFAULT_SAVE_DIR = os.curdir +DEFAULT_DEPRECATION_MESSAGE = "ez_setup.py is deprecated and when using it setuptools will be pinned to {0} since it's the last version that supports setuptools self upgrade/installation, check https://github.com/pypa/setuptools/issues/581 for more info; use pip to install setuptools" +MEANINGFUL_INVALID_ZIP_ERR_MSG = 'Maybe {0} is corrupted, delete it and try again.' -def _python_cmd(*args): - args = (sys.executable,) + args - return subprocess.call(args) == 0 +log.warn(DEFAULT_DEPRECATION_MESSAGE.format(DEFAULT_VERSION)) -def _check_call_py24(cmd, *args, **kwargs): - res = subprocess.call(cmd, *args, **kwargs) - class CalledProcessError(Exception): - pass - if not res == 0: - msg = "Command '%s' return non-zero exit status %d" % (cmd, res) - raise CalledProcessError(msg) - -vars(subprocess).setdefault('check_call', _check_call_py24) - +def _python_cmd(*args): + """ + Execute a command. -def _install(tarball, install_args=()): - # extracting the tarball - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - tar = tarfile.open(tarball) - _extractall(tar) - tar.close() + Return True if the command succeeded. + """ + args = (sys.executable,) + args + return subprocess.call(args) == 0 - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) +def _install(archive_filename, install_args=()): + """Install Setuptools.""" + with archive_context(archive_filename): # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): @@ -72,94 +64,167 @@ def _install(tarball, install_args=()): log.warn('See the error message above.') # exitcode will be 2 return 2 - finally: - os.chdir(old_wd) - shutil.rmtree(tmpdir) -def _build_egg(egg, tarball, to_dir): - # extracting the tarball +def _build_egg(egg, archive_filename, to_dir): + """Build Setuptools egg.""" + with archive_context(archive_filename): + # building an egg + log.warn('Building a Setuptools egg in %s', to_dir) + _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) + # returning the result + log.warn(egg) + if not os.path.exists(egg): + raise IOError('Could not build the egg.') + + +class ContextualZipFile(zipfile.ZipFile): + + """Supplement ZipFile class to support context manager for Python 2.6.""" + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def __new__(cls, *args, **kwargs): + """Construct a ZipFile or ContextualZipFile as appropriate.""" + if hasattr(zipfile.ZipFile, '__exit__'): + return zipfile.ZipFile(*args, **kwargs) + return super(ContextualZipFile, cls).__new__(cls) + + +@contextlib.contextmanager +def archive_context(filename): + """ + Unzip filename to a temporary directory, set to the cwd. + + The unzipped target is cleaned up after. + """ tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) - tar = tarfile.open(tarball) - _extractall(tar) - tar.close() + try: + with ContextualZipFile(filename) as archive: + archive.extractall() + except zipfile.BadZipfile as err: + if not err.args: + err.args = ('', ) + err.args = err.args + ( + MEANINGFUL_INVALID_ZIP_ERR_MSG.format(filename), + ) + raise # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) - - # building an egg - log.warn('Building a Setuptools egg in %s', to_dir) - _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) + yield finally: os.chdir(old_wd) shutil.rmtree(tmpdir) - # returning the result - log.warn(egg) - if not os.path.exists(egg): - raise IOError('Could not build the egg.') def _do_download(version, download_base, to_dir, download_delay): - egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' - % (version, sys.version_info[0], sys.version_info[1])) + """Download Setuptools.""" + py_desig = 'py{sys.version_info[0]}.{sys.version_info[1]}'.format(sys=sys) + tp = 'setuptools-{version}-{py_desig}.egg' + egg = os.path.join(to_dir, tp.format(**locals())) if not os.path.exists(egg): - tarball = download_setuptools(version, download_base, - to_dir, download_delay) - _build_egg(egg, tarball, to_dir) + archive = download_setuptools(version, download_base, + to_dir, download_delay) + _build_egg(egg, archive, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: - del sys.modules['pkg_resources'] + _unload_pkg_resources() import setuptools setuptools.bootstrap_install_from = egg -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15): - # making sure we use the absolute path +def use_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, download_delay=15): + """ + Ensure that a setuptools version is installed. + + Return None. Raise SystemExit if the requested version + or later cannot be installed. + """ to_dir = os.path.abspath(to_dir) - was_imported = 'pkg_resources' in sys.modules or \ - 'setuptools' in sys.modules + + # prior to importing, capture the module state for + # representative modules. + rep_modules = 'pkg_resources', 'setuptools' + imported = set(sys.modules).intersection(rep_modules) + try: import pkg_resources - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: pkg_resources.require("setuptools>=" + version) + # a suitable version is already installed return - except pkg_resources.VersionConflict: - e = sys.exc_info()[1] - if was_imported: - sys.stderr.write( - "The required version of setuptools (>=%s) is not available,\n" - "and can't be installed while this script is running. Please\n" - "install a more recent version first, using\n" - "'easy_install -U setuptools'." - "\n\n(Currently using %r)\n" % (version, e.args[0])) - sys.exit(2) - else: - del pkg_resources, sys.modules['pkg_resources'] # reload ok - return _do_download(version, download_base, to_dir, - download_delay) + except ImportError: + # pkg_resources not available; setuptools is not installed; download + pass except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, - download_delay) + # no version of setuptools was found; allow download + pass + except pkg_resources.VersionConflict as VC_err: + if imported: + _conflict_bail(VC_err, version) + + # otherwise, unload pkg_resources to allow the downloaded version to + # take precedence. + del pkg_resources + _unload_pkg_resources() + + return _do_download(version, download_base, to_dir, download_delay) + + +def _conflict_bail(VC_err, version): + """ + Setuptools was imported prior to invocation, so it is + unsafe to unload it. Bail out. + """ + conflict_tmpl = textwrap.dedent(""" + The required version of setuptools (>={version}) is not available, + and can't be installed while this script is running. Please + install a more recent version first, using + 'easy_install -U setuptools'. + + (Currently using {VC_err.args[0]!r}) + """) + msg = conflict_tmpl.format(**locals()) + sys.stderr.write(msg) + sys.exit(2) + + +def _unload_pkg_resources(): + sys.meta_path = [ + importer + for importer in sys.meta_path + if importer.__class__.__module__ != 'pkg_resources.extern' + ] + del_modules = [ + name for name in sys.modules + if name.startswith('pkg_resources') + ] + for mod_name in del_modules: + del sys.modules[mod_name] def _clean_check(cmd, target): """ - Run the command to download target. If the command fails, clean up before - re-raising the error. + Run the command to download target. + + If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) @@ -171,54 +236,53 @@ def _clean_check(cmd, target): def download_file_powershell(url, target): """ - Download the file at url to target using Powershell (which will validate - trust). Raise an exception if the command cannot complete. + Download the file at url to target using Powershell. + + Powershell will validate trust. + Raise an exception if the command cannot complete. """ target = os.path.abspath(target) + ps_cmd = ( + "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " + "[System.Net.CredentialCache]::DefaultCredentials; " + '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' + % locals() + ) cmd = [ 'powershell', '-Command', - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), + ps_cmd, ] _clean_check(cmd, target) def has_powershell(): + """Determine if Powershell is available.""" if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: + except Exception: return False - finally: - devnull.close() return True - - download_file_powershell.viable = has_powershell def download_file_curl(url, target): - cmd = ['curl', url, '--silent', '--output', target] + cmd = ['curl', url, '--location', '--silent', '--output', target] _clean_check(cmd, target) def has_curl(): cmd = ['curl', '--version'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: + except Exception: return False - finally: - devnull.close() return True - - download_file_curl.viable = has_curl @@ -229,67 +293,50 @@ def download_file_wget(url, target): def has_wget(): cmd = ['wget', '--version'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: + except Exception: return False - finally: - devnull.close() return True - - download_file_wget.viable = has_wget def download_file_insecure(url, target): - """ - Use Python to download the file, even though it cannot authenticate the - connection. - """ + """Use Python to download the file, without connection authentication.""" + src = urlopen(url) try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen - src = dst = None - try: - src = urlopen(url) - # Read/write all in one block, so we don't create a corrupt file - # if the download is interrupted. + # Read all the data in one block. data = src.read() - dst = open(target, "wb") - dst.write(data) finally: - if src: - src.close() - if dst: - dst.close() - + src.close() + # Write all the data in one block to avoid creating a partial file. + with open(target, "wb") as dst: + dst.write(data) download_file_insecure.viable = lambda: True def get_best_downloader(): - downloaders = [ + downloaders = ( download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, - ] - - for dl in downloaders: - if dl.viable(): - return dl + ) + viable_downloaders = (dl for dl in downloaders if dl.viable()) + return next(viable_downloaders, None) -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15, - downloader_factory=get_best_downloader): - """Download setuptools from a specified location and return its filename +def download_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, delay=15, + downloader_factory=get_best_downloader): + """ + Download setuptools from a specified location and return its filename. `version` should be a valid setuptools version number that is available - as an egg for download under the `download_base` URL (which should end + as an sdist for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. @@ -299,9 +346,9 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, """ # making sure we use the absolute path to_dir = os.path.abspath(to_dir) - tgz_name = "setuptools-%s.tar.gz" % version - url = download_base + tgz_name - saveto = os.path.join(to_dir, tgz_name) + zip_name = "setuptools-%s.zip" % version + url = download_base + zip_name + saveto = os.path.join(to_dir, zip_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() @@ -309,74 +356,21 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, return os.path.realpath(saveto) -def _extractall(self, path=".", members=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). - """ - import copy - import operator - from tarfile import ExtractError - directories = [] - - if members is None: - members = self - - for tarinfo in members: - if tarinfo.isdir(): - # Extract directories with a safe mode. - directories.append(tarinfo) - tarinfo = copy.copy(tarinfo) - tarinfo.mode = 448 # decimal for oct 0700 - self.extract(tarinfo, path) - - # Reverse sort directories. - if sys.version_info < (2, 4): - def sorter(dir1, dir2): - return cmp(dir1.name, dir2.name) - directories.sort(sorter) - directories.reverse() - else: - directories.sort(key=operator.attrgetter('name'), reverse=True) - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) - try: - self.chown(tarinfo, dirpath) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError: - e = sys.exc_info()[1] - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - def _build_install_args(options): """ - Build the arguments to 'python setup.py install' on the setuptools package + Build the arguments to 'python setup.py install' on the setuptools package. + + Returns list of command line arguments. """ - install_args = [] - if options.user_install: - if sys.version_info < (2, 6): - log.warn("--user requires Python 2.6 or later") - raise SystemExit(1) - install_args.append('--user') - return install_args + return ['--user'] if options.user_install else [] def _parse_args(): - """ - Parse the command line for options - """ + """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, - help='install in user site package (requires Python 2.6 or later)') + help='install in user site package') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, @@ -386,18 +380,35 @@ def _parse_args(): const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) + parser.add_option( + '--version', help="Specify which version to download", + default=DEFAULT_VERSION, + ) + parser.add_option( + '--to-dir', + help="Directory to save (and re-use) package", + default=DEFAULT_SAVE_DIR, + ) options, args = parser.parse_args() # positional arguments are ignored return options -def main(version=DEFAULT_VERSION): - """Install or upgrade setuptools and EasyInstall""" - options = _parse_args() - tarball = download_setuptools(download_base=options.download_base, - downloader_factory=options.downloader_factory) - return _install(tarball, _build_install_args(options)) +def _download_args(options): + """Return args for download_setuptools function from cmdline args.""" + return dict( + version=options.version, + download_base=options.download_base, + downloader_factory=options.downloader_factory, + to_dir=options.to_dir, + ) + +def main(): + """Install or upgrade setuptools and EasyInstall.""" + options = _parse_args() + archive = download_setuptools(**_download_args(options)) + return _install(archive, _build_install_args(options)) if __name__ == '__main__': sys.exit(main())