Merge pull request #3877 from Carreau/implement-pep-503-data-requires

Implement pep 503 data-requires-python

CHANGES.txt

@@ -10,6 +10,10 @@
 * Fix regression in pip freeze: when there is more than one git remote,
   priority is given to the remote named origin (:issue:`3616`)
 
+* Implementation of pep-503 ``data-requires-python``. When this field is
+  present for a release link, pip will ignore the download when
+  installing to a Python version that doesn't satisfy the requirement.
+
 * Pip wheel now works on editable packages too (it was only working on
   editable dependencies before); this allows running pip wheel on the result
   of pip freeze in presence of editable requirements (:issue:`3291`)

pip/baseparser.py

@@ -113,6 +113,7 @@ def expand_default(self, option):
 
 
 class CustomOptionParser(optparse.OptionParser):
+
     def insert_option_group(self, idx, *args, **kwargs):
         """Insert an OptionGroup at a given position."""
         group = self.add_option_group(*args, **kwargs)

pip/index.py

@@ -22,6 +22,7 @@
 )
 from pip.utils.deprecation import RemovedInPip9Warning, RemovedInPip10Warning
 from pip.utils.logging import indent_log
+from pip.utils.packaging import check_requires_python
 from pip.exceptions import (
     DistributionNotFound, BestVersionAlreadyInstalled, InvalidWheelFilename,
     UnsupportedWheel,
@@ -32,7 +33,9 @@
 from pip._vendor import html5lib, requests, six
 from pip._vendor.packaging.version import parse as parse_version
 from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.packaging import specifiers
 from pip._vendor.requests.exceptions import SSLError
+from pip._vendor.distlib.compat import unescape
 
 
 __all__ = ['FormatControl', 'fmt_ctl_handle_mutual_exclude', 'PackageFinder']
@@ -640,6 +643,18 @@ def _link_package_versions(self, link, search):
                 self._log_skipped_link(
                     link, 'Python version is incorrect')
                 return
+        try:
+            support_this_python = check_requires_python(link.requires_python)
+        except specifiers.InvalidSpecifier:
+            logger.debug("Package %s has an invalid Requires-Python entry: %s",
+                         link.filename, link.requires_python)
+            support_this_python = True
+
+        if not support_this_python:
+            logger.debug("The package %s is incompatible with the python"
+                         "version in use. Acceptable python versions are:%s",
+                         link, link.requires_python)
+            return
         logger.debug('Found link %s, version: %s', link, version)
 
         return InstallationCandidate(search.supplied, version, link)
@@ -828,7 +843,9 @@ def links(self):
                 url = self.clean_link(
                     urllib_parse.urljoin(self.base_url, href)
                 )
-                yield Link(url, self)
+                pyrequire = anchor.get('data-requires-python')
+                pyrequire = unescape(pyrequire) if pyrequire else None
+                yield Link(url, self, requires_python=pyrequire)
 
     _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
 
@@ -842,18 +859,35 @@ def clean_link(self, url):
 
 class Link(object):
 
-    def __init__(self, url, comes_from=None):
+    def __init__(self, url, comes_from=None, requires_python=None):
+        """
+        Object representing a parsed link from https://pypi.python.org/simple/*
+
+        url:
+            url of the resource pointed to (href of the link)
+        comes_from:
+            instance of HTMLPage where the link was found, or string.
+        requires_python:
+            String containing the `Requires-Python` metadata field, specified
+            in PEP 345. This may be specified by a data-requires-python
+            attribute in the HTML link tag, as described in PEP 503.
+        """
 
         # url can be a UNC windows share
         if url.startswith('\\\\'):
             url = path_to_url(url)
 
         self.url = url
         self.comes_from = comes_from
+        self.requires_python = requires_python if requires_python else None
 
     def __str__(self):
+        if self.requires_python:
+            rp = ' (requires-python:%s)' % self.requires_python
+        else:
+            rp = ''
         if self.comes_from:
-            return '%s (from %s)' % (self.url, self.comes_from)
+            return '%s (from %s)%s' % (self.url, self.comes_from, rp)
         else:
             return str(self.url)
 

pip/utils/packaging.py

@@ -0,0 +1,28 @@
+from __future__ import absolute_import
+import logging
+import sys
+
+from pip._vendor.packaging import specifiers
+from pip._vendor.packaging import version
+
+logger = logging.getLogger(__name__)
+
+
+def check_requires_python(requires_python):
+    """
+    Check if the python version in use match the `requires_python` specifier.
+
+    Returns `True` if the version of python in use matches the requirement.
+    Returns `False` if the version of python in use does not matches the
+    requirement.
+
+    Raises an InvalidSpecifier if `requires_python` have an invalid format.
+    """
+    if requires_python is None:
+        # The package provides no information
+        return True
+    requires_python_specifier = specifiers.SpecifierSet(requires_python)
+
+    # We only use major.minor.micro
+    python_version = version.parse('.'.join(map(str, sys.version_info[:3])))
+    return python_version in requires_python_specifier

tests/data/indexes/datarequire/fakepackage/index.html

@@ -0,0 +1,8 @@
+<html><head><title>Links for fakepackage</title><meta name="api-version" value="2" /></head><body><h1>Links for fakepackage</h1>
+    <a data-requires-python=''                href="/fakepackage-1.0.0.tar.gz#md5=00000000000000000000000000000000" rel="internal">fakepackage-1.0.0.tar.gz</a><br/>
+    <a data-requires-python='&lt;2.7'         href="/fakepackage-2.6.0.tar.gz#md5=00000000000000000000000000000000" rel="internal">fakepackage-2.6.0.tar.gz</a><br/>
+    <a data-requires-python='&gt;=2.7,&lt;3'  href="/fakepackage-2.7.0.tar.gz#md5=00000000000000000000000000000000" rel="internal">fakepackage-2.7.0.tar.gz</a><br/>
+    <a data-requires-python='&gt;=3.3'        href="/fakepackage-3.3.0.tar.gz#md5=00000000000000000000000000000000" rel="internal">fakepackage-3.3.0.tar.gz</a><br/>
+    <a data-requires-python='&gt;&lt;X.y.z'   href="/fakepackage-9.9.9.tar.gz#md5=00000000000000000000000000000000" rel="internal">fakepackage-9.9.9.tar.gz</a><br/>
+</body></html>
+

tests/scripts/test_all_pip.py

@@ -38,10 +38,9 @@ def main(args=None):
         print('Downloading pending list')
         projects = all_projects()
         print('Found %s projects' % len(projects))
-        f = open(pending_fn, 'w')
-        for name in projects:
-            f.write(name + '\n')
-        f.close()
+        with open(pending_fn, 'w') as f:
+            for name in projects:
+                f.write(name + '\n')
     print('Starting testing...')
     while os.stat(pending_fn).st_size:
         _test_packages(output, pending_fn)

tests/unit/test_finder.py

@@ -1,4 +1,5 @@
 import pytest
+import sys
 
 import pip.wheel
 import pip.pep425tags
@@ -365,6 +366,34 @@ def test_finder_only_installs_stable_releases(data):
         assert link.url == "https://foo/bar-1.0.tar.gz"
 
 
+def test_finder_only_installs_data_require(data):
+    """
+    Test whether the PackageFinder understand data-python-requires
+
+    This can optionally be exposed by a simple-repository to tell which
+    distribution are compatible with which version of Python by adding a
+    data-python-require to the anchor links.
+
+    See pep 503 for more informations.
+    """
+
+    # using a local index (that has pre & dev releases)
+    finder = PackageFinder([],
+                           [data.index_url("datarequire")],
+                           session=PipSession())
+    links = finder.find_all_candidates("fakepackage")
+
+    expected = ['1.0.0', '9.9.9']
+    if sys.version_info < (2, 7):
+        expected.append('2.6.0')
+    elif (2, 7) < sys.version_info < (3,):
+        expected.append('2.7.0')
+    elif sys.version_info > (3, 3):
+        expected.append('3.3.0')
+
+    assert set([str(v.version) for v in links]) == set(expected)
+
+
 def test_finder_installs_pre_releases(data):
     """
     Test PackageFinder finds pre-releases if asked to.