From c2101982cb5b28963fe22792cbb30ce50988ab8d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 Jul 2016 15:10:57 -0700 Subject: [PATCH 1/9] Draft implementation of pep-503 data-requires-python This allows pip to understand the `data-requires-python` metadata information that can be set on a simple repository. This allows pip to ignore any release or file that would not be compatible with the current Python version even before trying to download and install this version. Relevant extract of pep 503 at the time of this writing. A repository MAY include a data-requires-python attribute on a file link. This exposes the Requires-Python metadata field, specified in PEP 345 , for the corresponding release. Where this is present, installer tools SHOULD ignore the download when installing to a Python version that doesn't satisfy the requirement. For example: ... In the attribute value, < and > have to be HTML encoded as < and > , respectively. This can mostly be used to, for example, mark a new sdist of a new package version as requires-python >3.4, and not fail to install or upgrade on users systems. This will require extra patches to PyPI-legacy and warehouse to be usable. Though releasing a version of pip that understand this feature is necessary to have wide adoption at the time when these metadata get actually published. --- pip/index.py | 21 ++++++++++++++++++--- pip/utils/packaging.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 pip/utils/packaging.py diff --git a/pip/index.py b/pip/index.py index f8a298e9f0d..950809ce0a5 100644 --- a/pip/index.py +++ b/pip/index.py @@ -22,6 +22,7 @@ ) from pip.utils.deprecation import RemovedInPip9Warning, RemovedInPip10Warning from pip.utils.logging import indent_log +from pip.utils.packaging import check_requires_python from pip.exceptions import ( DistributionNotFound, BestVersionAlreadyInstalled, InvalidWheelFilename, UnsupportedWheel, @@ -33,6 +34,7 @@ from pip._vendor.packaging.version import parse as parse_version from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.requests.exceptions import SSLError +from pip._vendor.distlib.compat import unescape __all__ = ['FormatControl', 'fmt_ctl_handle_mutual_exclude', 'PackageFinder'] @@ -640,6 +642,11 @@ def _link_package_versions(self, link, search): self._log_skipped_link( link, 'Python version is incorrect') return + + if not check_requires_python(link.requires_python): + print('check on ', link, "Showed it's not compatible with python version in use:", link.requires_python) + return + #import ipdb; ipdb.set_trace() logger.debug('Found link %s, version: %s', link, version) return InstallationCandidate(search.supplied, version, link) @@ -828,7 +835,7 @@ def links(self): url = self.clean_link( urllib_parse.urljoin(self.base_url, href) ) - yield Link(url, self) + yield Link(url, self, requires_python=anchor.get('data-requires-python')) _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) @@ -842,7 +849,7 @@ def clean_link(self, url): class Link(object): - def __init__(self, url, comes_from=None): + def __init__(self, url, comes_from=None, requires_python=None): # url can be a UNC windows share if url.startswith('\\\\'): @@ -850,10 +857,18 @@ def __init__(self, url, comes_from=None): self.url = url self.comes_from = comes_from + if not requires_python: + self.requires_python = None + else: + self.requires_python = unescape(requires_python) def __str__(self): + if self.requires_python: + rp = ' (requires-python:%s)' % self.requires_python + else: + rp = '' if self.comes_from: - return '%s (from %s)' % (self.url, self.comes_from) + return '%s (from %s)%s' % (self.url, self.comes_from, rp) else: return str(self.url) diff --git a/pip/utils/packaging.py b/pip/utils/packaging.py new file mode 100644 index 00000000000..9167a1c3f89 --- /dev/null +++ b/pip/utils/packaging.py @@ -0,0 +1,36 @@ +from __future__ import absolute_import + +import logging +import sys + +from pip._vendor import pkg_resources +from pip._vendor.packaging import specifiers +from pip._vendor.packaging import version + +logger = logging.getLogger(__name__) + + +def get_metadata(dist): + if (isinstance(dist, pkg_resources.DistInfoDistribution) and + dist.has_metadata('METADATA')): + return dist.get_metadata('METADATA') + elif dist.has_metadata('PKG-INFO'): + return dist.get_metadata('PKG-INFO') + + +def check_requires_python(requires_python): + if requires_python is None: + # The package provides no information + return True + try: + requires_python_specifier = specifiers.SpecifierSet(requires_python) + except specifiers.InvalidSpecifier as e: + logger.debug( + "Package %s has an invalid Requires-Python entry - %s" % ( + requires_python, e)) + return ValueError('Wrong Specifier') + + # We only use major.minor.micro + python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) + if python_version not in requires_python_specifier: + return False From dbcefb77785ae9c25e178e18709468f31601271c Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Tue, 26 Jul 2016 15:56:34 -0700 Subject: [PATCH 2/9] fixed debug log message --- pip/index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pip/index.py b/pip/index.py index 950809ce0a5..13fea33b3f9 100644 --- a/pip/index.py +++ b/pip/index.py @@ -644,9 +644,8 @@ def _link_package_versions(self, link, search): return if not check_requires_python(link.requires_python): - print('check on ', link, "Showed it's not compatible with python version in use:", link.requires_python) + logger.debug('The package ', link, "is incompatible with the python version in use. Acceptable python versions are:", link.requires_python) return - #import ipdb; ipdb.set_trace() logger.debug('Found link %s, version: %s', link, version) return InstallationCandidate(search.supplied, version, link) From 86b80cc55ac1590cd769e3791407d051bf8ce223 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 Jul 2016 15:48:06 -0700 Subject: [PATCH 3/9] Add DocString to Link. --- pip/index.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pip/index.py b/pip/index.py index 13fea33b3f9..74ba8c4bce0 100644 --- a/pip/index.py +++ b/pip/index.py @@ -849,6 +849,19 @@ def clean_link(self, url): class Link(object): def __init__(self, url, comes_from=None, requires_python=None): + """ + Object representing a parsed link from https://pypi.python.org/simple// + + url: + url of the resource pointed to (href of the link) + comes_form: + + requires_python: + String containing the `Requires-Python` metadata field, specified + in PEP 345. This is to understand pep 503. The `requires_python` + string will be unescaped as pep 503 requires `<` and `>` to be + escaped, then stored under the `requires_python` attribute. + """ # url can be a UNC windows share if url.startswith('\\\\'): From f7373a4c2adc74fc2ff18a7c441a978b6982df89 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 Jul 2016 16:36:48 -0700 Subject: [PATCH 4/9] Update check_requires_python and describe behavior in docstring --- pip/utils/packaging.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pip/utils/packaging.py b/pip/utils/packaging.py index 9167a1c3f89..9d5f6c52232 100644 --- a/pip/utils/packaging.py +++ b/pip/utils/packaging.py @@ -19,6 +19,13 @@ def get_metadata(dist): def check_requires_python(requires_python): + """ + Check if the python version in used match the `requires_python` specifier passed. + + Return `True` if the version of python in use matches the requirement. + Return `False` if the version of python in use does not matches the requirement. + Raises an InvalidSpecifier if `requires_python` have an invalid format. + """ if requires_python is None: # The package provides no information return True @@ -28,9 +35,9 @@ def check_requires_python(requires_python): logger.debug( "Package %s has an invalid Requires-Python entry - %s" % ( requires_python, e)) - return ValueError('Wrong Specifier') + raise specifiers.InvalidSpecifier(*e.args) # We only use major.minor.micro python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) - if python_version not in requires_python_specifier: - return False + return python_version in requires_python_specifier + From 17d830f9620e120c307243063d09bf4115c32e12 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 Jul 2016 16:43:52 -0700 Subject: [PATCH 5/9] Remove get_metadata --- pip/utils/packaging.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pip/utils/packaging.py b/pip/utils/packaging.py index 9d5f6c52232..efd3f214a1a 100644 --- a/pip/utils/packaging.py +++ b/pip/utils/packaging.py @@ -1,23 +1,12 @@ from __future__ import absolute_import - import logging import sys -from pip._vendor import pkg_resources from pip._vendor.packaging import specifiers from pip._vendor.packaging import version logger = logging.getLogger(__name__) - -def get_metadata(dist): - if (isinstance(dist, pkg_resources.DistInfoDistribution) and - dist.has_metadata('METADATA')): - return dist.get_metadata('METADATA') - elif dist.has_metadata('PKG-INFO'): - return dist.get_metadata('PKG-INFO') - - def check_requires_python(requires_python): """ Check if the python version in used match the `requires_python` specifier passed. From 4c31a55f639fcf6c66ca14bdf47165a49b449d66 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 Jul 2016 18:26:22 -0700 Subject: [PATCH 6/9] 'fix pep8' --- pip/index.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pip/index.py b/pip/index.py index 74ba8c4bce0..380bf041e04 100644 --- a/pip/index.py +++ b/pip/index.py @@ -644,7 +644,9 @@ def _link_package_versions(self, link, search): return if not check_requires_python(link.requires_python): - logger.debug('The package ', link, "is incompatible with the python version in use. Acceptable python versions are:", link.requires_python) + logger.debug("The package %s is incompatible with the python" + "version in use. Acceptable python versions are:%s", + link, link.requires_python) return logger.debug('Found link %s, version: %s', link, version) @@ -834,7 +836,8 @@ def links(self): url = self.clean_link( urllib_parse.urljoin(self.base_url, href) ) - yield Link(url, self, requires_python=anchor.get('data-requires-python')) + pyrequire = anchor.get('data-requires-python') + yield Link(url, self, requires_python=pyrequire) _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) @@ -850,7 +853,7 @@ class Link(object): def __init__(self, url, comes_from=None, requires_python=None): """ - Object representing a parsed link from https://pypi.python.org/simple// + Object representing a parsed link from https://pypi.python.org/simple/* url: url of the resource pointed to (href of the link) From 1d10fca6fc5d5b15fd0aca8b593c48c876b370c8 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 27 Jul 2016 12:15:24 -0700 Subject: [PATCH 7/9] Fix pep-8, assume work on this Python or invalid specifiers. move the unescape outside of Link class. reraise using raise that is available on Python 2.6 --- pip/index.py | 17 ++++++++++------- pip/utils/packaging.py | 13 +++++++------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pip/index.py b/pip/index.py index 380bf041e04..19eee25fbf4 100644 --- a/pip/index.py +++ b/pip/index.py @@ -33,6 +33,7 @@ from pip._vendor import html5lib, requests, six from pip._vendor.packaging.version import parse as parse_version from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.packaging import specifiers from pip._vendor.requests.exceptions import SSLError from pip._vendor.distlib.compat import unescape @@ -642,8 +643,12 @@ def _link_package_versions(self, link, search): self._log_skipped_link( link, 'Python version is incorrect') return + try: + support_this_python = check_requires_python(link.requires_python) + except specifiers.InvalidSpecifier: + support_this_python = True - if not check_requires_python(link.requires_python): + if not support_this_python: logger.debug("The package %s is incompatible with the python" "version in use. Acceptable python versions are:%s", link, link.requires_python) @@ -836,7 +841,7 @@ def links(self): url = self.clean_link( urllib_parse.urljoin(self.base_url, href) ) - pyrequire = anchor.get('data-requires-python') + pyrequire = unescape(anchor.get('data-requires-python')) yield Link(url, self, requires_python=pyrequire) _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) @@ -857,13 +862,11 @@ def __init__(self, url, comes_from=None, requires_python=None): url: url of the resource pointed to (href of the link) - comes_form: + comes_from: requires_python: String containing the `Requires-Python` metadata field, specified - in PEP 345. This is to understand pep 503. The `requires_python` - string will be unescaped as pep 503 requires `<` and `>` to be - escaped, then stored under the `requires_python` attribute. + in PEP 345. """ # url can be a UNC windows share @@ -875,7 +878,7 @@ def __init__(self, url, comes_from=None, requires_python=None): if not requires_python: self.requires_python = None else: - self.requires_python = unescape(requires_python) + self.requires_python = requires_python def __str__(self): if self.requires_python: diff --git a/pip/utils/packaging.py b/pip/utils/packaging.py index efd3f214a1a..096156b2eda 100644 --- a/pip/utils/packaging.py +++ b/pip/utils/packaging.py @@ -7,13 +7,15 @@ logger = logging.getLogger(__name__) + def check_requires_python(requires_python): """ - Check if the python version in used match the `requires_python` specifier passed. + Check if the python version in used match the `requires_python` specifier. Return `True` if the version of python in use matches the requirement. - Return `False` if the version of python in use does not matches the requirement. - Raises an InvalidSpecifier if `requires_python` have an invalid format. + Return `False` if the version of python in use does not matches the + requirement. Raises an InvalidSpecifier if `requires_python` have an + invalid format. """ if requires_python is None: # The package provides no information @@ -23,10 +25,9 @@ def check_requires_python(requires_python): except specifiers.InvalidSpecifier as e: logger.debug( "Package %s has an invalid Requires-Python entry - %s" % ( - requires_python, e)) - raise specifiers.InvalidSpecifier(*e.args) + requires_python, e)) + raise # We only use major.minor.micro python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) return python_version in requires_python_specifier - From d4e22ea097e7e90c2e6f88a6412bbe924d5353dd Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 28 Jul 2016 12:11:53 -0700 Subject: [PATCH 8/9] Add tests --- pip/baseparser.py | 1 + pip/index.py | 15 +++++++------ pip/utils/packaging.py | 19 ++++++---------- .../datarequire/fakepackage/index.html | 8 +++++++ tests/scripts/test_all_pip.py | 7 +++--- tests/unit/test_finder.py | 22 +++++++++++++++++++ 6 files changed, 49 insertions(+), 23 deletions(-) create mode 100644 tests/data/indexes/datarequire/fakepackage/index.html diff --git a/pip/baseparser.py b/pip/baseparser.py index 9b53aca160a..2dd4533016b 100644 --- a/pip/baseparser.py +++ b/pip/baseparser.py @@ -113,6 +113,7 @@ def expand_default(self, option): class CustomOptionParser(optparse.OptionParser): + def insert_option_group(self, idx, *args, **kwargs): """Insert an OptionGroup at a given position.""" group = self.add_option_group(*args, **kwargs) diff --git a/pip/index.py b/pip/index.py index 19eee25fbf4..15cf460e76b 100644 --- a/pip/index.py +++ b/pip/index.py @@ -646,6 +646,8 @@ def _link_package_versions(self, link, search): try: support_this_python = check_requires_python(link.requires_python) except specifiers.InvalidSpecifier: + logger.debug("Package %s has an invalid Requires-Python entry: %s", + link.filename, link.requires_python) support_this_python = True if not support_this_python: @@ -841,7 +843,8 @@ def links(self): url = self.clean_link( urllib_parse.urljoin(self.base_url, href) ) - pyrequire = unescape(anchor.get('data-requires-python')) + pyrequire = anchor.get('data-requires-python') + pyrequire = unescape(pyrequire) if pyrequire else None yield Link(url, self, requires_python=pyrequire) _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) @@ -863,10 +866,11 @@ def __init__(self, url, comes_from=None, requires_python=None): url: url of the resource pointed to (href of the link) comes_from: - + instance of HTMLPage where the link was found, or string. requires_python: String containing the `Requires-Python` metadata field, specified - in PEP 345. + in PEP 345. This may be specified by a data-requires-python + attribute in the HTML link tag, as described in PEP 503. """ # url can be a UNC windows share @@ -875,10 +879,7 @@ def __init__(self, url, comes_from=None, requires_python=None): self.url = url self.comes_from = comes_from - if not requires_python: - self.requires_python = None - else: - self.requires_python = requires_python + self.requires_python = requires_python if requires_python else None def __str__(self): if self.requires_python: diff --git a/pip/utils/packaging.py b/pip/utils/packaging.py index 096156b2eda..96318867d2a 100644 --- a/pip/utils/packaging.py +++ b/pip/utils/packaging.py @@ -10,23 +10,18 @@ def check_requires_python(requires_python): """ - Check if the python version in used match the `requires_python` specifier. + Check if the python version in use match the `requires_python` specifier. - Return `True` if the version of python in use matches the requirement. - Return `False` if the version of python in use does not matches the - requirement. Raises an InvalidSpecifier if `requires_python` have an - invalid format. + Returns `True` if the version of python in use matches the requirement. + Returns `False` if the version of python in use does not matches the + requirement. + + Raises an InvalidSpecifier if `requires_python` have an invalid format. """ if requires_python is None: # The package provides no information return True - try: - requires_python_specifier = specifiers.SpecifierSet(requires_python) - except specifiers.InvalidSpecifier as e: - logger.debug( - "Package %s has an invalid Requires-Python entry - %s" % ( - requires_python, e)) - raise + requires_python_specifier = specifiers.SpecifierSet(requires_python) # We only use major.minor.micro python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) diff --git a/tests/data/indexes/datarequire/fakepackage/index.html b/tests/data/indexes/datarequire/fakepackage/index.html new file mode 100644 index 00000000000..8f61127bc28 --- /dev/null +++ b/tests/data/indexes/datarequire/fakepackage/index.html @@ -0,0 +1,8 @@ +Links for fakepackage

Links for fakepackage

+ fakepackage-1.0.0.tar.gz
+ fakepackage-2.6.0.tar.gz
+ fakepackage-2.7.0.tar.gz
+ fakepackage-3.3.0.tar.gz
+ fakepackage-9.9.9.tar.gz
+ + diff --git a/tests/scripts/test_all_pip.py b/tests/scripts/test_all_pip.py index 95fc8723c53..c0e91ff41b1 100644 --- a/tests/scripts/test_all_pip.py +++ b/tests/scripts/test_all_pip.py @@ -38,10 +38,9 @@ def main(args=None): print('Downloading pending list') projects = all_projects() print('Found %s projects' % len(projects)) - f = open(pending_fn, 'w') - for name in projects: - f.write(name + '\n') - f.close() + with open(pending_fn, 'w') as f: + for name in projects: + f.write(name + '\n') print('Starting testing...') while os.stat(pending_fn).st_size: _test_packages(output, pending_fn) diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 48265e98d89..741b2a3b4bf 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -1,4 +1,5 @@ import pytest +import sys import pip.wheel import pip.pep425tags @@ -365,6 +366,27 @@ def test_finder_only_installs_stable_releases(data): assert link.url == "https://foo/bar-1.0.tar.gz" +def test_finder_only_installs_data_require(data): + """ + """ + + # using a local index (that has pre & dev releases) + finder = PackageFinder([], + [data.index_url("datarequire")], + session=PipSession()) + links = finder.find_all_candidates("fakepackage") + + expected = ['1.0.0', '9.9.9'] + if sys.version_info < (2, 7): + expected.append('2.6.0') + elif (2, 7) < sys.version_info < (3,): + expected.append('2.7.0') + elif sys.version_info > (3, 3): + expected.append('3.3.0') + + assert set([str(v.version) for v in links]) == set(expected) + + def test_finder_installs_pre_releases(data): """ Test PackageFinder finds pre-releases if asked to. From bfd8794aad00251e3ed0db23a249171a2bb09f18 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 Jul 2016 09:41:10 -0700 Subject: [PATCH 9/9] Set test docstring ad add to CHANGES.txt --- CHANGES.txt | 4 ++++ tests/unit/test_finder.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index a5189c4a39b..08cfc061ed6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,6 +8,10 @@ * Fix regression in pip freeze: when there is more than one git remote, priority is given to the remote named origin (:issue:`3616`) +* Implementation of pep-503 ``data-requires-python``. When this field is + present for a release link, pip will ignore the download when + installing to a Python version that doesn't satisfy the requirement. + * Pip wheel now works on editable packages too (it was only working on editable dependencies before); this allows running pip wheel on the result of pip freeze in presence of editable requirements (:issue:`3291`) diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 741b2a3b4bf..793cf7a7c29 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -368,6 +368,13 @@ def test_finder_only_installs_stable_releases(data): def test_finder_only_installs_data_require(data): """ + Test whether the PackageFinder understand data-python-requires + + This can optionally be exposed by a simple-repository to tell which + distribution are compatible with which version of Python by adding a + data-python-require to the anchor links. + + See pep 503 for more informations. """ # using a local index (that has pre & dev releases)