From e483f533fa8c2c95ebb5cb22cdce89885c740dfb Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 13 Aug 2021 18:08:57 +0800 Subject: [PATCH 1/2] Switch install scheme backend to sysconfig This is only done for Python 3.10+ so we don't disrupt too many existing pip installations. --- news/10358.removal.rst | 4 ++ setup.cfg | 1 + src/pip/_internal/locations/__init__.py | 40 ++++++++++++++----- src/pip/_internal/operations/install/wheel.py | 5 +-- tests/conftest.py | 4 ++ tests/unit/test_locations.py | 4 +- 6 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 news/10358.removal.rst diff --git a/news/10358.removal.rst b/news/10358.removal.rst new file mode 100644 index 00000000000..727631cd322 --- /dev/null +++ b/news/10358.removal.rst @@ -0,0 +1,4 @@ +On Python 3.10 or later, the installation scheme backend has been changed to use +``sysconfig``. This is to anticipate the deprecation of ``distutils`` in Python +3.10, and its scheduled removal in 3.12. For compatibility considerations, pip +installations running on Python 3.9 or lower will continue to use ``distutils``. diff --git a/setup.cfg b/setup.cfg index d5dfb587e7b..be58a126bca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,7 @@ follow_imports = skip addopts = --ignore src/pip/_vendor --ignore tests/tests_cache -r aR --color=yes markers = network: tests that need network + incompatible_with_sysconfig incompatible_with_test_venv incompatible_with_venv no_auto_tempdir_manager diff --git a/src/pip/_internal/locations/__init__.py b/src/pip/_internal/locations/__init__.py index d74857b2ac9..4f3fdf4a939 100644 --- a/src/pip/_internal/locations/__init__.py +++ b/src/pip/_internal/locations/__init__.py @@ -45,6 +45,8 @@ _PLATLIBDIR: str = getattr(sys, "platlibdir", "lib") +_USE_SYSCONFIG = sys.version_info >= (3, 10) + def _looks_like_bpo_44860() -> bool: """The resolution to bpo-44860 will change this incorrect platlib. @@ -190,7 +192,7 @@ def get_scheme( isolated: bool = False, prefix: Optional[str] = None, ) -> Scheme: - old = _distutils.get_scheme( + new = _sysconfig.get_scheme( dist_name, user=user, home=home, @@ -198,7 +200,10 @@ def get_scheme( isolated=isolated, prefix=prefix, ) - new = _sysconfig.get_scheme( + if _USE_SYSCONFIG: + return new + + old = _distutils.get_scheme( dist_name, user=user, home=home, @@ -335,8 +340,11 @@ def get_scheme( def get_bin_prefix() -> str: - old = _distutils.get_bin_prefix() new = _sysconfig.get_bin_prefix() + if _USE_SYSCONFIG: + return new + + old = _distutils.get_bin_prefix() if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix"): _log_context() return old @@ -365,8 +373,11 @@ def _looks_like_deb_system_dist_packages(value: str) -> bool: def get_purelib() -> str: """Return the default pure-Python lib location.""" - old = _distutils.get_purelib() new = _sysconfig.get_purelib() + if _USE_SYSCONFIG: + return new + + old = _distutils.get_purelib() if _looks_like_deb_system_dist_packages(old): return old if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="purelib"): @@ -376,8 +387,11 @@ def get_purelib() -> str: def get_platlib() -> str: """Return the default platform-shared lib location.""" - old = _distutils.get_platlib() new = _sysconfig.get_platlib() + if _USE_SYSCONFIG: + return new + + old = _distutils.get_platlib() if _looks_like_deb_system_dist_packages(old): return old if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="platlib"): @@ -385,10 +399,20 @@ def get_platlib() -> str: return old +def _deduplicated(v1: str, v2: str) -> List[str]: + """Deduplicate values from a list.""" + if v1 == v2: + return [v1] + return [v1, v2] + + def get_prefixed_libs(prefix: str) -> List[str]: """Return the lib locations under ``prefix``.""" - old_pure, old_plat = _distutils.get_prefixed_libs(prefix) new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix) + if _USE_SYSCONFIG: + return _deduplicated(new_pure, new_plat) + + old_pure, old_plat = _distutils.get_prefixed_libs(prefix) warned = [ _warn_if_mismatch( @@ -405,6 +429,4 @@ def get_prefixed_libs(prefix: str) -> List[str]: if any(warned): _log_context(prefix=prefix) - if old_pure == old_plat: - return [old_pure] - return [old_pure, old_plat] + return _deduplicated(old_pure, old_plat) diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 1d3c303369c..29437da2ab6 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -628,10 +628,7 @@ def pyc_output_path(path: str) -> str: with warnings.catch_warnings(): warnings.filterwarnings("ignore") for path in pyc_source_file_paths(): - # Python 2's `compileall.compile_file` requires a str in - # error cases, so we must convert to the native type. - path_arg = ensure_str(path, encoding=sys.getfilesystemencoding()) - success = compileall.compile_file(path_arg, force=True, quiet=True) + success = compileall.compile_file(path, force=True, quiet=True) if success: pyc_path = pyc_output_path(path) assert os.path.exists(pyc_path) diff --git a/tests/conftest.py b/tests/conftest.py index af567e5f1f6..520d0cc72f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ from setuptools.wheel import Wheel from pip._internal.cli.main import main as pip_entry_point +from pip._internal.locations import _USE_SYSCONFIG from pip._internal.utils.temp_dir import global_tempdir_manager from tests.lib import DATA_DIR, SRC_DIR, PipTestEnvironment, TestData from tests.lib.certs import make_tls_cert, serialize_cert, serialize_key @@ -77,6 +78,9 @@ def pytest_collection_modifyitems(config, items): ): item.add_marker(pytest.mark.skip("Incompatible with venv")) + if item.get_closest_marker("incompatible_with_sysconfig") and _USE_SYSCONFIG: + item.add_marker(pytest.mark.skip("Incompatible with sysconfig")) + module_path = os.path.relpath( item.module.__file__, os.path.commonprefix([__file__, item.module.__file__]), diff --git a/tests/unit/test_locations.py b/tests/unit/test_locations.py index b605f2c6a57..31308cf80fb 100644 --- a/tests/unit/test_locations.py +++ b/tests/unit/test_locations.py @@ -96,6 +96,7 @@ def test_root_modifies_appropriately(self, monkeypatch): expected = os.path.join(root, path[1:]) assert os.path.abspath(root_scheme[key]) == expected + @pytest.mark.incompatible_with_sysconfig @pytest.mark.incompatible_with_venv def test_distutils_config_file_read(self, tmpdir, monkeypatch): # This deals with nt/posix path differences @@ -116,10 +117,11 @@ def test_distutils_config_file_read(self, tmpdir, monkeypatch): scheme = _get_scheme_dict("example") assert scheme["scripts"] == install_scripts + @pytest.mark.incompatible_with_sysconfig @pytest.mark.incompatible_with_venv # when we request install-lib, we should install everything (.py & # .so) into that path; i.e. ensure platlib & purelib are set to - # this path + # this path. sysconfig does not support this. def test_install_lib_takes_precedence(self, tmpdir, monkeypatch): # This deals with nt/posix path differences install_lib = os.path.normcase( From d1d9bc5849c200e11456342017f8832631f85bf1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 21 Aug 2021 00:47:57 +0800 Subject: [PATCH 2/2] Add Python 3.10 workaround --- tests/unit/test_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 247e8d6643c..e21c0ec2e68 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -422,7 +422,7 @@ def test_install_prefix(self, data, tmpdir): self.name, user=False, home=None, - root=tmpdir, + root=str(tmpdir), # Casting needed for CPython 3.10+. See GH-10358. isolated=False, prefix=prefix, )