diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index fdefbe1a431..157be28b678 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -4,7 +4,7 @@ about: Create a report to help us improve --- **Environment** diff --git a/NEWS.rst b/NEWS.rst index 755ca709ee5..aaacfffb4ef 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,34 @@ .. towncrier release notes start +20.2.1 (2020-08-04) +=================== + +Features +-------- + +- Ignore require-virtualenv in ``pip list`` (`#8603 `_) + +Bug Fixes +--------- + +- Correctly find already-installed distributions with dot (``.``) in the name + and uninstall them when needed. (`#8645 `_) +- Trace a better error message on installation failure due to invalid ``.data`` + files in wheels. (`#8654 `_) +- Fix SVN version detection for alternative SVN distributions. (`#8665 `_) +- New resolver: Correctly include the base package when specified with extras + in ``--no-deps`` mode. (`#8677 `_) +- Use UTF-8 to handle ZIP archive entries on Python 2 according to PEP 427, so + non-ASCII paths can be resolved as expected. (`#8684 `_) + +Improved Documentation +---------------------- + +- Add details on old resolver deprecation and removal to migration documentation. (`#8371 `_) +- Fix feature flag name in docs. (`#8660 `_) + + 20.2 (2020-07-29) ================= @@ -43,6 +71,7 @@ Features break. More details about how to test and migrate, and how to report issues, at :ref:`Resolver changes 2020` . Maintainers are preparing to release pip 20.3, with the new resolver on by default, in October. (`#6536 `_) +- Introduce a new ResolutionImpossible error, raised when pip encounters un-satisfiable dependency conflicts (`#8546 `_, `#8377 `_) - Add a subcommand ``debug`` to ``pip config`` to list available configuration sources and the key-value pairs defined in them. (`#6741 `_) - Warn if index pages have unexpected content-type (`#6754 `_) - Allow specifying ``--prefer-binary`` option in a requirements file (`#7693 `_) @@ -92,7 +121,7 @@ Improved Documentation - Fix pip config docstring so that the subcommands render correctly in the docs (`#8072 `_) - replace links to the old pypa-dev mailing list with https://mail.python.org/mailman3/lists/distutils-sig.python.org/ (`#8353 `_) - Fix example for defining multiple values for options which support them (`#8373 `_) -- Add documentation that helps the user fix dependency conflicts (`#8459 `_) +- Add documentation for the ResolutionImpossible error that helps the user fix dependency conflicts (`#8459 `_) - Add feature flags to docs (`#8512 `_) - Document how to install package extras from git branch and source distributions. (`#8576 `_) diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst index cbfbce4adf9..44197955e8e 100644 --- a/docs/html/development/release-process.rst +++ b/docs/html/development/release-process.rst @@ -2,6 +2,7 @@ Release process =============== +.. _`Release Cadence`: Release Cadence =============== @@ -72,6 +73,8 @@ only bugs will be considered, and merged (subject to normal review processes). Note that there may be delays due to the lack of developer resources for reviewing such pull requests. +.. _`Feature Flags`: + Feature Flags ============= diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 702a97d0ca1..a03ec164c44 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -783,7 +783,7 @@ specified packages due to conflicting dependencies (a ``ResolutionImpossible`` error). This documentation is specific to the new resolver, which you can use -with the flag ``--unstable-feature=resolver``. +with the flag ``--use-feature=2020-resolver``. Understanding your error message -------------------------------- @@ -945,7 +945,7 @@ Unfortunately, **the pip team cannot provide support for individual dependency conflict errors**. Please *only* open a ticket on the `pip issue tracker`_ if you believe that your problem has exposed a bug in pip. -.. _dependency hell: https://en.wikipedia.org/wiki/Dependency_hell> +.. _dependency hell: https://en.wikipedia.org/wiki/Dependency_hell .. _Awesome Python: https://python.libhunt.com/ .. _Python user Discourse: https://discuss.python.org/c/users/7 .. _Python user forums: https://www.python.org/community/forums/ @@ -1029,13 +1029,13 @@ Changes to the pip dependency resolver in 20.2 (2020) ===================================================== pip 20.1 included an alpha version of the new resolver (hidden behind -an optional ``--unstable-feature=resolver`` flag). pip 20.2 includes a -robust beta of the new resolver (hidden behind an optional -``--use-feature=2020-resolver`` flag) that we encourage you to -test. We will continue to improve the pip dependency resolver in -response to testers' feedback. Please give us feedback through the -`resolver testing survey`_. This will help us prepare to release pip -20.3, with the new resolver on by default, in October. +an optional ``--unstable-feature=resolver`` flag). pip 20.2 removes +that flag, and includes a robust beta of the new resolver (hidden +behind an optional ``--use-feature=2020-resolver`` flag) that we +encourage you to test. We will continue to improve the pip dependency +resolver in response to testers' feedback. Please give us feedback +through the `resolver testing survey`_. This will help us prepare to +release pip 20.3, with the new resolver on by default, in October. Watch out for ------------- @@ -1223,18 +1223,37 @@ Specific things we'd love to get feedback on: Please let us know through the `resolver testing survey`_. +Deprecation timeline +-------------------- + +We plan for the resolver changeover to proceed as follows, using +:ref:`Feature Flags` and following our :ref:`Release Cadence`: + +* pip 20.2: a beta of the new resolver is available, opt-in, using + the flag ``--use-feature=2020-resolver``. pip defaults to + legacy behavior. + +* pip 20.3: pip defaults to the new resolver, but a user can opt-out + and choose the old resolver behavior, using the flag + ``--use-deprecated=legacy-resolver``. + +* pip 21.0: pip uses new resolver, and the old resolver is no longer + available. + +Since this work will not change user-visible behavior described in the +pip documentation, this change is not covered by the :ref:`Deprecation +Policy`. + Context and followup -------------------- As discussed in `our announcement on the PSF blog`_, the pip team are in the process of developing a new "dependency resolver" (the part of -pip that works out what to install based on your requirements). Since -this work will not change user-visible behavior described in the pip -documentation, this change is not covered by the :ref:`Deprecation -Policy`. +pip that works out what to install based on your requirements). We're tracking our rollout in :issue:`6536` and you can watch for -announcements on the `low-traffic packaging announcements list`_. +announcements on the `low-traffic packaging announcements list`_ and +`the official Python blog`_. .. _freeze: https://pip.pypa.io/en/latest/reference/pip_freeze/ .. _resolver testing survey: https://tools.simplysecure.org/survey/index.php?r=survey/index&sid=989272&lang=en @@ -1242,3 +1261,4 @@ announcements on the `low-traffic packaging announcements list`_. .. _tensorflow: https://pypi.org/project/tensorflow/ .. _low-traffic packaging announcements list: https://mail.python.org/mailman3/lists/pypi-announce.python.org/ .. _our survey on upgrades that create conflicts: https://docs.google.com/forms/d/e/1FAIpQLSeBkbhuIlSofXqCyhi3kGkLmtrpPOEBwr6iJA6SzHdxWKfqdA/viewform +.. _the official Python blog: https://blog.python.org/ diff --git a/src/pip/__init__.py b/src/pip/__init__.py index b67e61d063e..5a2f3c31745 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -4,7 +4,7 @@ from typing import List, Optional -__version__ = "20.2" +__version__ = "20.3.dev0" def main(args=None): diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index a67d0f8d4ab..20e9bff2b71 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -39,6 +39,7 @@ class ListCommand(IndexGroupCommand): Packages are listed in a case-insensitive sorted order. """ + ignore_require_venv = True usage = """ %prog [options]""" diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index c2371bf5cd3..16be0d2972a 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -109,8 +109,10 @@ def read(self, size=-1): all bytes until EOF are returned. Fewer than size bytes may be returned if EOF is reached. """ + download_size = max(size, self._chunk_size) start, length = self.tell(), self._length - stop = start + size if 0 <= size <= length-start else length + stop = length if size < 0 else min(start+download_size, length) + start = max(0, stop-download_size) self._download(start, stop-1) return self._file.read(size) diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 8f73a88b074..e91b1b8d558 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -78,6 +78,7 @@ Union, cast, ) + from zipfile import ZipInfo from pip._vendor.pkg_resources import Distribution @@ -420,6 +421,15 @@ def __init__(self, src_record_path, dest_path, zip_file): self._zip_file = zip_file self.changed = False + def _getinfo(self): + # type: () -> ZipInfo + if not PY2: + return self._zip_file.getinfo(self.src_record_path) + # Python 2 does not expose a way to detect a ZIP's encoding, but the + # wheel specification (PEP 427) explicitly mandates that paths should + # use UTF-8, so we assume it is true. + return self._zip_file.getinfo(self.src_record_path.encode("utf-8")) + def save(self): # type: () -> None # directory creation is lazy and after file filtering @@ -439,11 +449,12 @@ def save(self): if os.path.exists(self.dest_path): os.unlink(self.dest_path) - with self._zip_file.open(self.src_record_path) as f: + zipinfo = self._getinfo() + + with self._zip_file.open(zipinfo) as f: with open(self.dest_path, "wb") as dest: shutil.copyfileobj(f, dest) - zipinfo = self._zip_file.getinfo(self.src_record_path) if zip_item_is_executable(zipinfo): set_extracted_file_to_default_mode_plus_executable(self.dest_path) @@ -583,8 +594,28 @@ def data_scheme_file_maker(zip_file, scheme): def make_data_scheme_file(record_path): # type: (RecordPath) -> File normed_path = os.path.normpath(record_path) - _, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2) - scheme_path = scheme_paths[scheme_key] + try: + _, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2) + except ValueError: + message = ( + "Unexpected file in {}: {!r}. .data directory contents" + " should be named like: '/'." + ).format(wheel_path, record_path) + raise InstallationError(message) + + try: + scheme_path = scheme_paths[scheme_key] + except KeyError: + valid_scheme_keys = ", ".join(sorted(scheme_paths)) + message = ( + "Unknown scheme key used in {}: {} (for file {!r}). .data" + " directory contents should be in subdirectories named" + " with a valid scheme key ({})" + ).format( + wheel_path, scheme_key, record_path, valid_scheme_keys + ) + raise InstallationError(message) + dest_path = os.path.join(scheme_path, dest_subpath) assert_no_path_traversal(scheme_path, dest_path) return ZipBackedFile(record_path, dest_path, zip_file) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 644930a1528..4759f4af6f0 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -429,25 +429,13 @@ def check_if_exists(self, use_user_site): """ if self.req is None: return - # get_distribution() will resolve the entire list of requirements - # anyway, and we've already determined that we need the requirement - # in question, so strip the marker so that we don't try to - # evaluate it. - no_marker = Requirement(str(self.req)) - no_marker.marker = None - - # pkg_resources uses the canonical name to look up packages, but - # the name passed passed to get_distribution is not canonicalized - # so we have to explicitly convert it to a canonical name - no_marker.name = canonicalize_name(no_marker.name) - try: - self.satisfied_by = pkg_resources.get_distribution(str(no_marker)) - except pkg_resources.DistributionNotFound: + existing_dist = get_distribution(self.req.name) + if not existing_dist: return - except pkg_resources.VersionConflict: - existing_dist = get_distribution( - self.req.name - ) + + existing_version = existing_dist.parsed_version + if not self.req.specifier.contains(existing_version, prereleases=True): + self.satisfied_by = None if use_user_site: if dist_in_usersite(existing_dist): self.should_reinstall = True @@ -461,11 +449,13 @@ def check_if_exists(self, use_user_site): else: self.should_reinstall = True else: - if self.editable and self.satisfied_by: + if self.editable: self.should_reinstall = True # when installing editables, nothing pre-existing should ever # satisfy self.satisfied_by = None + else: + self.satisfied_by = existing_dist # Things valid for wheels @property diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index a155a1101ad..9245747bf2b 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -69,8 +69,8 @@ def source_link(self): # type: () -> Optional[Link] raise NotImplementedError("Override in subclass") - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, with_requires): + # type: (bool) -> Iterable[Optional[Requirement]] raise NotImplementedError("Override in subclass") def get_install_requirement(self): diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index c289bb5839c..46cc7e7a236 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -275,8 +275,10 @@ def _get_requires_python_specifier(self): return None return spec - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, with_requires): + # type: (bool) -> Iterable[Optional[Requirement]] + if not with_requires: + return for r in self.dist.requires(): yield self._factory.make_requirement_from_spec(str(r), self._ireq) python_dep = self._factory.make_requires_python_requirement( @@ -420,8 +422,10 @@ def format_for_error(self): # type: () -> str return "{} {} (Installed)".format(self.name, self.version) - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, with_requires): + # type: (bool) -> Iterable[Optional[Requirement]] + if not with_requires: + return for r in self.dist.requires(): yield self._factory.make_requirement_from_spec(str(r), self._ireq) @@ -519,10 +523,16 @@ def source_link(self): # type: () -> Optional[Link] return self.base.source_link - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, with_requires): + # type: (bool) -> Iterable[Optional[Requirement]] factory = self.base._factory + # Add a dependency on the exact base + # (See note 2b in the class docstring) + yield factory.make_requirement_from_candidate(self.base) + if not with_requires: + return + # The user may have specified extras that the candidate doesn't # support. We ignore any unsupported extras here. valid_extras = self.extras.intersection(self.base.dist.extras) @@ -535,10 +545,6 @@ def iter_dependencies(self): extra ) - # Add a dependency on the exact base - # (See note 2b in the class docstring) - yield factory.make_requirement_from_candidate(self.base) - for r in self.base.dist.requires(valid_extras): requirement = factory.make_requirement_from_spec( str(r), self.base._ireq, valid_extras, @@ -585,8 +591,8 @@ def format_for_error(self): # type: () -> str return "Python {}".format(self.version) - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, with_requires): + # type: (bool) -> Iterable[Optional[Requirement]] return () def get_install_requirement(self): diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index bd7e3efd9d3..dab23aa09d1 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -453,7 +453,7 @@ def describe_trigger(parent): logger.info(msg) return DistributionNotFound( - "ResolutionImpossible For help visit: " - "https://pip.pypa.io/en/stable/user_guide/" + "ResolutionImpossible: for help visit " + "https://pip.pypa.io/en/latest/user_guide/" "#fixing-conflicting-dependencies" ) diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 72f16205981..b2eb9d06ea5 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -145,6 +145,9 @@ def is_satisfied_by(self, requirement, candidate): def get_dependencies(self, candidate): # type: (Candidate) -> Sequence[Requirement] - if self._ignore_dependencies: - return [] - return [r for r in candidate.iter_dependencies() if r is not None] + with_requires = not self._ignore_dependencies + return [ + r + for r in candidate.iter_dependencies(with_requires) + if r is not None + ] diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index 14825f791a4..ab134970b05 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -213,6 +213,8 @@ def call_vcs_version(self): # compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0 # svn, version 1.7.14 (r1542130) # compiled Mar 28 2018, 08:49:13 on x86_64-pc-linux-gnu + # svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0) + # compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2 version_prefix = 'svn, version ' version = self.run_command(['--version']) @@ -220,7 +222,7 @@ def call_vcs_version(self): return () version = version[len(version_prefix):].split()[0] - version_list = version.split('.') + version_list = version.partition('-')[0].split('.') try: parsed_version = tuple(map(int, version_list)) except ValueError: diff --git a/tests/functional/test_install_upgrade.py b/tests/functional/test_install_upgrade.py index e45bf31483e..02e221101c5 100644 --- a/tests/functional/test_install_upgrade.py +++ b/tests/functional/test_install_upgrade.py @@ -1,3 +1,4 @@ +import itertools import os import sys import textwrap @@ -7,6 +8,7 @@ from tests.lib import pyversion # noqa: F401 from tests.lib import assert_all_changes from tests.lib.local_repos import local_checkout +from tests.lib.wheel import make_wheel @pytest.mark.network @@ -439,3 +441,34 @@ def prep_ve(self, script, version, pip_src, distribute=False): cwd=pip_src, expect_stderr=True, ) + + +@pytest.mark.parametrize("req1, req2", list(itertools.product( + ["foo.bar", "foo_bar", "foo-bar"], ["foo.bar", "foo_bar", "foo-bar"], +))) +def test_install_find_existing_package_canonicalize(script, req1, req2): + """Ensure an already-installed dist is found no matter how the dist name + was normalized on installation. (pypa/pip#8645) + """ + # Create and install a package that's not available in the later stage. + req_container = script.scratch_path.joinpath("foo-bar") + req_container.mkdir() + req_path = make_wheel("foo_bar", "1.0").save_to_dir(req_container) + script.pip("install", "--no-index", req_path) + + # Depend on the previously installed, but now unavailable package. + pkg_container = script.scratch_path.joinpath("pkg") + pkg_container.mkdir() + make_wheel( + "pkg", + "1.0", + metadata_updates={"Requires-Dist": req2}, + ).save_to_dir(pkg_container) + + # Ensure the previously installed package can be correctly used to match + # the dependency. + result = script.pip( + "install", "--no-index", "--find-links", pkg_container, "pkg", + ) + satisfied_message = "Requirement already satisfied: {}".format(req2) + assert satisfied_message in result.stdout, str(result) diff --git a/tests/functional/test_install_wheel.py b/tests/functional/test_install_wheel.py index c53f13ca415..ad4e749676f 100644 --- a/tests/functional/test_install_wheel.py +++ b/tests/functional/test_install_wheel.py @@ -681,3 +681,36 @@ def test_correct_package_name_while_creating_wheel_bug(script, package_name): package = create_basic_wheel_for_package(script, package_name, '1.0') wheel_name = os.path.basename(package) assert wheel_name == 'simple_package-1.0-py2.py3-none-any.whl' + + +@pytest.mark.parametrize("name", ["purelib", "abc"]) +def test_wheel_with_file_in_data_dir_has_reasonable_error( + script, tmpdir, name +): + """Normally we expect entities in the .data directory to be in a + subdirectory, but if they are not then we should show a reasonable error + message that includes the path. + """ + wheel_path = make_wheel( + "simple", "0.1.0", extra_data_files={name: "hello world"} + ).save_to_dir(tmpdir) + + result = script.pip( + "install", "--no-index", str(wheel_path), expect_error=True + ) + assert "simple-0.1.0.data/{}".format(name) in result.stderr + + +def test_wheel_with_unknown_subdir_in_data_dir_has_reasonable_error( + script, tmpdir +): + wheel_path = make_wheel( + "simple", + "0.1.0", + extra_data_files={"unknown/hello.txt": "hello world"} + ).save_to_dir(tmpdir) + + result = script.pip( + "install", "--no-index", str(wheel_path), expect_error=True + ) + assert "simple-0.1.0.data/unknown/hello.txt" in result.stderr diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 590cb5c0b75..93598c36739 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -443,6 +443,9 @@ def test_subversion__call_vcs_version(): ('svn, version 1.10.3 (r1842928)\n' ' compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0', (1, 10, 3)), + ('svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)\n' + ' compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2', + (1, 12, 0)), ('svn, version 1.9.7 (r1800392)', (1, 9, 7)), ('svn, version 1.9.7a1 (r1800392)', ()), ('svn, version 1.9 (r1800392)', (1, 9)), diff --git a/tests/yaml/extras.yml b/tests/yaml/extras.yml index 6e2a1b17e7b..ac68fae4979 100644 --- a/tests/yaml/extras.yml +++ b/tests/yaml/extras.yml @@ -40,3 +40,10 @@ cases: - E 1.0.0 - F 1.0.0 skip: old +- + request: + - install: D[extra_1] + options: --no-deps + response: + - state: + - D 1.0.0