From 79cd5998aa617f772e6b905177c93cb1f55634ec Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 28 Jul 2022 12:10:50 +0100 Subject: [PATCH 01/17] Add a --python option --- src/pip/_internal/cli/cmdoptions.py | 8 ++++++++ src/pip/_internal/cli/main_parser.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 47ed92779e9..84e0e783869 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -189,6 +189,13 @@ class PipOption(Option): ), ) +python: Callable[..., Option] = partial( + Option, + "--python", + dest="python", + help="Run pip with the specified Python interpreter.", +) + verbose: Callable[..., Option] = partial( Option, "-v", @@ -1029,6 +1036,7 @@ def check_list_path_option(options: Values) -> None: debug_mode, isolated_mode, require_virtualenv, + python, verbose, version, quiet, diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 3666ab04ca6..8a79191c8b2 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -2,9 +2,11 @@ """ import os +import subprocess import sys from typing import List, Tuple +from pip._internal.build_env import _get_runnable_pip from pip._internal.cli import cmdoptions from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter from pip._internal.commands import commands_dict, get_similar_commands @@ -57,6 +59,19 @@ def parse_command(args: List[str]) -> Tuple[str, List[str]]: # args_else: ['install', '--user', 'INITools'] general_options, args_else = parser.parse_args(args) + # --python + if general_options.python: + if "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ: + pip_cmd = [ + general_options.python, + _get_runnable_pip(), + ] + pip_cmd.extend(args) + # Block recursing indefinitely + os.environ["_PIP_RUNNING_IN_SUBPROCESS"] = "1" + proc = subprocess.run(pip_cmd) + sys.exit(proc.returncode) + # --version if general_options.version: sys.stdout.write(parser.version) From 95cf55bf185b41ce029b5349a8a8f016d6c5e8a5 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 28 Jul 2022 14:31:10 +0100 Subject: [PATCH 02/17] Add a news file --- news/11320.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11320.feature.rst diff --git a/news/11320.feature.rst b/news/11320.feature.rst new file mode 100644 index 00000000000..028f16c2bcf --- /dev/null +++ b/news/11320.feature.rst @@ -0,0 +1 @@ +Add a ``--python`` option to specify the Python environment to be managed by pip. From 42eae5033e33aace218186f8b76b731771268bbd Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 28 Jul 2022 15:04:35 +0100 Subject: [PATCH 03/17] More flexible handling of the --python argument --- src/pip/_internal/cli/main_parser.py | 72 +++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 8a79191c8b2..967d568e22c 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -2,15 +2,17 @@ """ import os +import shutil import subprocess import sys -from typing import List, Tuple +from typing import List, Optional, Tuple from pip._internal.build_env import _get_runnable_pip from pip._internal.cli import cmdoptions from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter from pip._internal.commands import commands_dict, get_similar_commands from pip._internal.exceptions import CommandError +from pip._internal.utils.compat import WINDOWS from pip._internal.utils.misc import get_pip_version, get_prog __all__ = ["create_main_parser", "parse_command"] @@ -47,6 +49,44 @@ def create_main_parser() -> ConfigOptionParser: return parser +def identify_python_interpreter(python: str) -> Optional[str]: + if python == "python" or python == "py": + # Run the active Python. + # We have to be very careful here, because: + # + # 1. On Unix, "python" is probably correct but there is a "py" launcher. + # 2. On Windows, "py" is the best option if it's present. + # 3. On Windows without "py", "python" might work, but it might also + # be the shim that launches the Windows store to allow you to install + # Python. + # + # We go with getting py on Windows, and if it's not present or we're + # on Unix, get python. We don't worry about the launcher on Unix or + # the installer stub on Windows. + py = None + if WINDOWS: + py = shutil.which("py") + if py is None: + py = shutil.which("python") + if py: + return py + + # TODO: On Windows, `--python .venv/Scripts/python` won't pass the + # exists() check (no .exe extension supplied). But it's pretty + # obvious what the user intends. Should we allow this? + if os.path.exists(python): + if not os.path.isdir(python): + return python + # Might be a virtual environment + for exe in ("bin/python", "Scripts/python.exe"): + py = os.path.join(python, exe) + if os.path.exists(py): + return py + + # Could not find the interpreter specified + return None + + def parse_command(args: List[str]) -> Tuple[str, List[str]]: parser = create_main_parser() @@ -60,17 +100,25 @@ def parse_command(args: List[str]) -> Tuple[str, List[str]]: general_options, args_else = parser.parse_args(args) # --python - if general_options.python: - if "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ: - pip_cmd = [ - general_options.python, - _get_runnable_pip(), - ] - pip_cmd.extend(args) - # Block recursing indefinitely - os.environ["_PIP_RUNNING_IN_SUBPROCESS"] = "1" - proc = subprocess.run(pip_cmd) - sys.exit(proc.returncode) + if general_options.python and "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ: + # Re-invoke pip using the specified Python interpreter + interpreter = identify_python_interpreter(general_options.python) + if interpreter is None: + raise CommandError( + f"Could not locate Python interpreter {general_options.python}" + ) + + pip_cmd = [ + interpreter, + _get_runnable_pip(), + ] + pip_cmd.extend(args) + + # Set a flag so the child doesn't re-invoke itself, causing + # an infinite loop. + os.environ["_PIP_RUNNING_IN_SUBPROCESS"] = "1" + proc = subprocess.run(pip_cmd) + sys.exit(proc.returncode) # --version if general_options.version: From 78e7ea88e98a66a5e0d8dd6574ad3323e13c1a8e Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 29 Jul 2022 09:37:29 +0100 Subject: [PATCH 04/17] Make get_runnable_pip public --- src/pip/_internal/build_env.py | 4 ++-- src/pip/_internal/cli/main_parser.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 6d4f6a56eb7..6213eedd14a 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -39,7 +39,7 @@ def __init__(self, path: str) -> None: self.lib_dirs = get_prefixed_libs(path) -def _get_runnable_pip() -> str: +def get_runnable_pip() -> str: """Get a file to pass to a Python executable, to run the currently-running pip. This is used to run a pip subprocess, for installing requirements into the build @@ -194,7 +194,7 @@ def install_requirements( if not requirements: return self._install_requirements( - _get_runnable_pip(), + get_runnable_pip(), finder, requirements, prefix, diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 967d568e22c..61dc42a1298 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -7,7 +7,7 @@ import sys from typing import List, Optional, Tuple -from pip._internal.build_env import _get_runnable_pip +from pip._internal.build_env import get_runnable_pip from pip._internal.cli import cmdoptions from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter from pip._internal.commands import commands_dict, get_similar_commands @@ -110,7 +110,7 @@ def parse_command(args: List[str]) -> Tuple[str, List[str]]: pip_cmd = [ interpreter, - _get_runnable_pip(), + get_runnable_pip(), ] pip_cmd.extend(args) From 24c22a3e5d0ad30dcb6fabf68047185496bd21d8 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 29 Jul 2022 09:44:14 +0100 Subject: [PATCH 05/17] Check the argument to --python is executable --- src/pip/_internal/cli/main_parser.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 61dc42a1298..6502c567794 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -71,17 +71,20 @@ def identify_python_interpreter(python: str) -> Optional[str]: if py: return py - # TODO: On Windows, `--python .venv/Scripts/python` won't pass the - # exists() check (no .exe extension supplied). But it's pretty - # obvious what the user intends. Should we allow this? + # If the named file exists, and is executable, use it. + # If it's a directory, assume it's a virtual environment and + # look for the environment's Python executable. if os.path.exists(python): - if not os.path.isdir(python): + # Do the directory check first because directories can be executable + if os.path.isdir(python): + # bin/python for Unix, Scripts/python.exe for Windows + # Try both in case of odd cases like cygwin. + for exe in ("bin/python", "Scripts/python.exe"): + py = os.path.join(python, exe) + if os.path.exists(py): + return py + elif os.access(python, os.X_OK): return python - # Might be a virtual environment - for exe in ("bin/python", "Scripts/python.exe"): - py = os.path.join(python, exe) - if os.path.exists(py): - return py # Could not find the interpreter specified return None From 01e122ed4125673c77bfd2aced75e70f6dfa2a7c Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 29 Jul 2022 10:42:34 +0100 Subject: [PATCH 06/17] Add tests --- tests/functional/test_python_option.py | 33 ++++++++++++++++++++++++++ tests/unit/test_cmdoptions.py | 31 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tests/functional/test_python_option.py diff --git a/tests/functional/test_python_option.py b/tests/functional/test_python_option.py new file mode 100644 index 00000000000..4fafde2a8b2 --- /dev/null +++ b/tests/functional/test_python_option.py @@ -0,0 +1,33 @@ +import json +import os +from pathlib import Path +from venv import EnvBuilder + +from tests.lib import PipTestEnvironment, TestData + + +def test_python_interpreter( + script: PipTestEnvironment, + tmpdir: Path, + shared_data: TestData, +) -> None: + env_path = os.fsdecode(tmpdir / "venv") + env = EnvBuilder(with_pip=False) + env.create(env_path) + + result = script.pip("--python", env_path, "list", "--format=json") + assert json.loads(result.stdout) == [] + script.pip( + "--python", + env_path, + "install", + "-f", + shared_data.find_links, + "--no-index", + "simplewheel==1.0", + ) + result = script.pip("--python", env_path, "list", "--format=json") + assert json.loads(result.stdout) == [{"name": "simplewheel", "version": "1.0"}] + script.pip("--python", env_path, "uninstall", "simplewheel", "--yes") + result = script.pip("--python", env_path, "list", "--format=json") + assert json.loads(result.stdout) == [] diff --git a/tests/unit/test_cmdoptions.py b/tests/unit/test_cmdoptions.py index 1e5ef995cd0..d5b4813822f 100644 --- a/tests/unit/test_cmdoptions.py +++ b/tests/unit/test_cmdoptions.py @@ -1,8 +1,12 @@ +import os +from pathlib import Path from typing import Optional, Tuple +from venv import EnvBuilder import pytest from pip._internal.cli.cmdoptions import _convert_python_version +from pip._internal.cli.main_parser import identify_python_interpreter @pytest.mark.parametrize( @@ -29,3 +33,30 @@ def test_convert_python_version( ) -> None: actual = _convert_python_version(value) assert actual == expected, f"actual: {actual!r}" + + +def test_identify_python_interpreter_py(monkeypatch: pytest.MonkeyPatch) -> None: + def which(cmd: str) -> str: + assert cmd == "py" or cmd == "python" + return "dummy_value" + + monkeypatch.setattr("shutil.which", which) + assert identify_python_interpreter("py") == "dummy_value" + assert identify_python_interpreter("python") == "dummy_value" + + +def test_identify_python_interpreter_venv(tmpdir: Path) -> None: + env_path = tmpdir / "venv" + env = EnvBuilder(with_pip=False) + env.create(env_path) + + # Passing a virtual environment returns the Python executable + interp = identify_python_interpreter(os.fsdecode(env_path)) + assert interp is not None + assert Path(interp).exists() + + # Passing an executable returns it + assert identify_python_interpreter(interp) == interp + + # Passing a non-existent file returns None + assert identify_python_interpreter(str(tmpdir / "nonexistent")) is None From b1eb91204e0673a61d9a3203a49675be3307abd2 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 29 Jul 2022 10:57:00 +0100 Subject: [PATCH 07/17] Added documentation --- docs/html/user_guide.rst | 43 ++++++++++++++++++++++++++++++++++++++++ news/11320.feature.rst | 3 ++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 70a28ab9988..1f1a8660627 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -782,6 +782,49 @@ This is now covered in :doc:`../topics/repeatable-installs`. This is now covered in :doc:`../topics/dependency-resolution`. +.. _`Managing a different Python interpreter`: + +Managing a different Python interpreter +======================================= + +Occasionally, you may want to use pip to manage a Python installation other than +the one pip is installed into. In this case, you can use the ``--python`` option +to specify the interpreter you want to manage. This option can take one of three +values: + +#. The path to a Python executable. +#. The path to a virtual environment. +#. Either "py" or "python", referring to the currently active Python interpreter. + +In all 3 cases, pip will run exactly as if it had been invoked from that Python +environment. + +One example of where this might be useful is to manage a virtual environment +that does not have pip installed. + +.. tab:: Unix/macOS + + .. code-block:: console + + $ python -m venv .venv --without-pip + $ python -m pip --python .venv install SomePackage + [...] + Successfully installed SomePackage + +.. tab:: Windows + + .. code-block:: console + + C:\> py -m venv .venv --without-pip + C:\> py -m pip --python .venv install SomePackage + [...] + Successfully installed SomePackage + +You could also use ``--python .venv/bin/python`` (or on Windows, +``--python .venv\Scripts\python.exe``) if you wanted to be explicit, but the +virtual environment name is shorter and works exactly the same. + + .. _`Using pip from your program`: Using pip from your program diff --git a/news/11320.feature.rst b/news/11320.feature.rst index 028f16c2bcf..843eac7c9f4 100644 --- a/news/11320.feature.rst +++ b/news/11320.feature.rst @@ -1 +1,2 @@ -Add a ``--python`` option to specify the Python environment to be managed by pip. +Add a ``--python`` option to allow pip to manage Python environments other +than the one pip is installed in. From f86a140b124bef3c4a6beeaa5d6fff1a4cd62a18 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 29 Jul 2022 16:59:31 +0100 Subject: [PATCH 08/17] Move docs to a topic --- docs/html/topics/index.md | 1 + docs/html/topics/python-option.md | 38 +++++++++++++++++++++++++++ docs/html/user_guide.rst | 43 ------------------------------- 3 files changed, 39 insertions(+), 43 deletions(-) create mode 100644 docs/html/topics/python-option.md diff --git a/docs/html/topics/index.md b/docs/html/topics/index.md index 011205a111d..c5e4d36c95f 100644 --- a/docs/html/topics/index.md +++ b/docs/html/topics/index.md @@ -18,4 +18,5 @@ local-project-installs repeatable-installs secure-installs vcs-support +python-option ``` diff --git a/docs/html/topics/python-option.md b/docs/html/topics/python-option.md new file mode 100644 index 00000000000..242784dfbe8 --- /dev/null +++ b/docs/html/topics/python-option.md @@ -0,0 +1,38 @@ +# Managing a different Python interpreter + + +Occasionally, you may want to use pip to manage a Python installation other than +the one pip is installed into. In this case, you can use the `--python` option +to specify the interpreter you want to manage. This option can take one of three +values: + +1. The path to a Python executable. +2. The path to a virtual environment. +3. Either "py" or "python", referring to the currently active Python interpreter. + +In all 3 cases, pip will run exactly as if it had been invoked from that Python +environment. + +One example of where this might be useful is to manage a virtual environment +that does not have pip installed. + +````{tab} Unix/macOS +```{code-block} console +$ python -m venv .venv --without-pip +$ python -m pip --python .venv install SomePackage +[...] +Successfully installed SomePackage +``` +```` +````{tab} Windows +```{code-block} console +C:\> py -m venv .venv --without-pip +C:\> py -m pip --python .venv install SomePackage +[...] +Successfully installed SomePackage +``` +```` + +You could also use `--python .venv/bin/python` (or on Windows, +`--python .venv\Scripts\python.exe`) if you wanted to be explicit, but the +virtual environment name is shorter and works exactly the same. diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 1f1a8660627..70a28ab9988 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -782,49 +782,6 @@ This is now covered in :doc:`../topics/repeatable-installs`. This is now covered in :doc:`../topics/dependency-resolution`. -.. _`Managing a different Python interpreter`: - -Managing a different Python interpreter -======================================= - -Occasionally, you may want to use pip to manage a Python installation other than -the one pip is installed into. In this case, you can use the ``--python`` option -to specify the interpreter you want to manage. This option can take one of three -values: - -#. The path to a Python executable. -#. The path to a virtual environment. -#. Either "py" or "python", referring to the currently active Python interpreter. - -In all 3 cases, pip will run exactly as if it had been invoked from that Python -environment. - -One example of where this might be useful is to manage a virtual environment -that does not have pip installed. - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m venv .venv --without-pip - $ python -m pip --python .venv install SomePackage - [...] - Successfully installed SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m venv .venv --without-pip - C:\> py -m pip --python .venv install SomePackage - [...] - Successfully installed SomePackage - -You could also use ``--python .venv/bin/python`` (or on Windows, -``--python .venv\Scripts\python.exe``) if you wanted to be explicit, but the -virtual environment name is shorter and works exactly the same. - - .. _`Using pip from your program`: Using pip from your program From b5afdd604831f985427880537d37eb7a35addaa1 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 29 Jul 2022 12:54:13 +0100 Subject: [PATCH 09/17] Fix test to cater for packages leaked into venv --- tests/functional/test_python_option.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_python_option.py b/tests/functional/test_python_option.py index 4fafde2a8b2..8bf16d7a56b 100644 --- a/tests/functional/test_python_option.py +++ b/tests/functional/test_python_option.py @@ -11,12 +11,17 @@ def test_python_interpreter( tmpdir: Path, shared_data: TestData, ) -> None: - env_path = os.fsdecode(tmpdir / "venv") + env_path = os.fspath(tmpdir / "venv") env = EnvBuilder(with_pip=False) env.create(env_path) result = script.pip("--python", env_path, "list", "--format=json") - assert json.loads(result.stdout) == [] + before = json.loads(result.stdout) + + # Ideally we would assert that before==[], but there's a problem in CI + # that means this isn't true. See https://github.com/pypa/pip/pull/11326 + # for details. + script.pip( "--python", env_path, @@ -26,8 +31,11 @@ def test_python_interpreter( "--no-index", "simplewheel==1.0", ) + result = script.pip("--python", env_path, "list", "--format=json") - assert json.loads(result.stdout) == [{"name": "simplewheel", "version": "1.0"}] + installed = json.loads(result.stdout) + assert {"name": "simplewheel", "version": "1.0"} in installed + script.pip("--python", env_path, "uninstall", "simplewheel", "--yes") result = script.pip("--python", env_path, "list", "--format=json") - assert json.loads(result.stdout) == [] + assert json.loads(result.stdout) == before From 61249ed9ee1811ef2693195e21432ebba738574a Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sat, 30 Jul 2022 16:55:48 +0100 Subject: [PATCH 10/17] Update docs/html/topics/python-option.md Co-authored-by: Pradyun Gedam --- docs/html/topics/python-option.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/html/topics/python-option.md b/docs/html/topics/python-option.md index 242784dfbe8..877f78b3041 100644 --- a/docs/html/topics/python-option.md +++ b/docs/html/topics/python-option.md @@ -16,22 +16,12 @@ environment. One example of where this might be useful is to manage a virtual environment that does not have pip installed. -````{tab} Unix/macOS -```{code-block} console +```{pip-cli} $ python -m venv .venv --without-pip -$ python -m pip --python .venv install SomePackage +$ pip --python .venv install SomePackage [...] Successfully installed SomePackage ``` -```` -````{tab} Windows -```{code-block} console -C:\> py -m venv .venv --without-pip -C:\> py -m pip --python .venv install SomePackage -[...] -Successfully installed SomePackage -``` -```` You could also use `--python .venv/bin/python` (or on Windows, `--python .venv\Scripts\python.exe`) if you wanted to be explicit, but the From d0b5a8f75dbac35896a394ab27fff5378ee23baf Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sat, 30 Jul 2022 16:56:09 +0100 Subject: [PATCH 11/17] Update docs/html/topics/python-option.md Co-authored-by: Pradyun Gedam --- docs/html/topics/python-option.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/html/topics/python-option.md b/docs/html/topics/python-option.md index 877f78b3041..435d8ed6774 100644 --- a/docs/html/topics/python-option.md +++ b/docs/html/topics/python-option.md @@ -1,5 +1,7 @@ # Managing a different Python interpreter +```{versionadded} 22.3 +``` Occasionally, you may want to use pip to manage a Python installation other than the one pip is installed into. In this case, you can use the `--python` option From b6be01aee8be13844fa054a5f8c56fa062cc2c21 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 31 Jul 2022 11:50:29 +0100 Subject: [PATCH 12/17] Catch errors from running the subprocess --- src/pip/_internal/cli/main_parser.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 6502c567794..06a61305d05 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -120,8 +120,13 @@ def parse_command(args: List[str]) -> Tuple[str, List[str]]: # Set a flag so the child doesn't re-invoke itself, causing # an infinite loop. os.environ["_PIP_RUNNING_IN_SUBPROCESS"] = "1" - proc = subprocess.run(pip_cmd) - sys.exit(proc.returncode) + returncode = 0 + try: + proc = subprocess.run(pip_cmd) + returncode = proc.returncode + except (subprocess.SubprocessError, OSError) as exc: + raise CommandError(f"Failed to run pip under {interpreter}: {exc}") + sys.exit(returncode) # --version if general_options.version: From 333389133a9fc5f07280a1025a466b137386b18a Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 31 Jul 2022 12:12:49 +0100 Subject: [PATCH 13/17] Check python version in __pip-runner__.py --- setup.py | 2 ++ src/pip/__pip-runner__.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/setup.py b/setup.py index 9b7fdeb1134..2179d34d2bf 100644 --- a/setup.py +++ b/setup.py @@ -81,5 +81,7 @@ def get_version(rel_path: str) -> str: ], }, zip_safe=False, + # NOTE: python_requires is duplicated in __pip-runner__.py. + # When changing this value, please change the other copy as well. python_requires=">=3.7", ) diff --git a/src/pip/__pip-runner__.py b/src/pip/__pip-runner__.py index 280e99f2f08..28e4399b054 100644 --- a/src/pip/__pip-runner__.py +++ b/src/pip/__pip-runner__.py @@ -12,6 +12,8 @@ from typing import Optional, Sequence, Union PIP_SOURCES_ROOT = dirname(dirname(__file__)) +# Copied from setup.py +PYTHON_REQUIRES = ">=3.7" class PipImportRedirectingFinder: @@ -30,8 +32,22 @@ def find_spec( return spec +def check_python_version() -> None: + # Import here to ensure the imports happen after the sys.meta_path change. + from pip._vendor.packaging.specifiers import SpecifierSet + from pip._vendor.packaging.version import Version + + py_ver = Version("{0.major}.{0.minor}.{0.micro}".format(sys.version_info)) + if py_ver not in SpecifierSet(PYTHON_REQUIRES): + raise SystemExit( + f"This version of pip does not support python {py_ver} " + f"(requires {PYTHON_REQUIRES})" + ) + + # TODO https://github.com/pypa/pip/issues/11294 sys.meta_path.insert(0, PipImportRedirectingFinder()) assert __name__ == "__main__", "Cannot run __pip-runner__.py as a non-main module" +check_python_version() runpy.run_module("pip", run_name="__main__", alter_sys=True) From 4dc35b7399cee3668caf31ba200d3dcfc2bd7579 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 31 Jul 2022 16:01:57 +0100 Subject: [PATCH 14/17] Skip the executable check, as subprocess.run will catch this --- src/pip/_internal/cli/main_parser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 06a61305d05..548174d8dfe 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -71,11 +71,10 @@ def identify_python_interpreter(python: str) -> Optional[str]: if py: return py - # If the named file exists, and is executable, use it. + # If the named file exists, use it. # If it's a directory, assume it's a virtual environment and # look for the environment's Python executable. if os.path.exists(python): - # Do the directory check first because directories can be executable if os.path.isdir(python): # bin/python for Unix, Scripts/python.exe for Windows # Try both in case of odd cases like cygwin. @@ -83,7 +82,7 @@ def identify_python_interpreter(python: str) -> Optional[str]: py = os.path.join(python, exe) if os.path.exists(py): return py - elif os.access(python, os.X_OK): + else: return python # Could not find the interpreter specified From ebe491a82a13e6610697b22be00db363fb5ff5e3 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 1 Aug 2022 10:49:32 +0100 Subject: [PATCH 15/17] Get rid of the --python python/py shortcuts --- src/pip/_internal/cli/main_parser.py | 23 ----------------------- tests/unit/test_cmdoptions.py | 12 +----------- 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 548174d8dfe..5ade356b9c2 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -2,7 +2,6 @@ """ import os -import shutil import subprocess import sys from typing import List, Optional, Tuple @@ -12,7 +11,6 @@ from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter from pip._internal.commands import commands_dict, get_similar_commands from pip._internal.exceptions import CommandError -from pip._internal.utils.compat import WINDOWS from pip._internal.utils.misc import get_pip_version, get_prog __all__ = ["create_main_parser", "parse_command"] @@ -50,27 +48,6 @@ def create_main_parser() -> ConfigOptionParser: def identify_python_interpreter(python: str) -> Optional[str]: - if python == "python" or python == "py": - # Run the active Python. - # We have to be very careful here, because: - # - # 1. On Unix, "python" is probably correct but there is a "py" launcher. - # 2. On Windows, "py" is the best option if it's present. - # 3. On Windows without "py", "python" might work, but it might also - # be the shim that launches the Windows store to allow you to install - # Python. - # - # We go with getting py on Windows, and if it's not present or we're - # on Unix, get python. We don't worry about the launcher on Unix or - # the installer stub on Windows. - py = None - if WINDOWS: - py = shutil.which("py") - if py is None: - py = shutil.which("python") - if py: - return py - # If the named file exists, use it. # If it's a directory, assume it's a virtual environment and # look for the environment's Python executable. diff --git a/tests/unit/test_cmdoptions.py b/tests/unit/test_cmdoptions.py index d5b4813822f..8c33ca8c18d 100644 --- a/tests/unit/test_cmdoptions.py +++ b/tests/unit/test_cmdoptions.py @@ -35,23 +35,13 @@ def test_convert_python_version( assert actual == expected, f"actual: {actual!r}" -def test_identify_python_interpreter_py(monkeypatch: pytest.MonkeyPatch) -> None: - def which(cmd: str) -> str: - assert cmd == "py" or cmd == "python" - return "dummy_value" - - monkeypatch.setattr("shutil.which", which) - assert identify_python_interpreter("py") == "dummy_value" - assert identify_python_interpreter("python") == "dummy_value" - - def test_identify_python_interpreter_venv(tmpdir: Path) -> None: env_path = tmpdir / "venv" env = EnvBuilder(with_pip=False) env.create(env_path) # Passing a virtual environment returns the Python executable - interp = identify_python_interpreter(os.fsdecode(env_path)) + interp = identify_python_interpreter(os.fspath(env_path)) assert interp is not None assert Path(interp).exists() From b8aa21b5759c55bf53f69c185b6193a19e82cd20 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 3 Aug 2022 10:05:25 +0100 Subject: [PATCH 16/17] Revert __pip-runner__.py changes --- setup.py | 2 -- src/pip/__pip-runner__.py | 16 ---------------- 2 files changed, 18 deletions(-) diff --git a/setup.py b/setup.py index 2179d34d2bf..9b7fdeb1134 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,5 @@ def get_version(rel_path: str) -> str: ], }, zip_safe=False, - # NOTE: python_requires is duplicated in __pip-runner__.py. - # When changing this value, please change the other copy as well. python_requires=">=3.7", ) diff --git a/src/pip/__pip-runner__.py b/src/pip/__pip-runner__.py index 41d7fe00474..14026c0d131 100644 --- a/src/pip/__pip-runner__.py +++ b/src/pip/__pip-runner__.py @@ -12,8 +12,6 @@ from typing import Optional, Sequence, Union PIP_SOURCES_ROOT = dirname(dirname(__file__)) -# Copied from setup.py -PYTHON_REQUIRES = ">=3.7" class PipImportRedirectingFinder: @@ -32,21 +30,7 @@ def find_spec( return spec -def check_python_version() -> None: - # Import here to ensure the imports happen after the sys.meta_path change. - from pip._vendor.packaging.specifiers import SpecifierSet - from pip._vendor.packaging.version import Version - - py_ver = Version("{0.major}.{0.minor}.{0.micro}".format(sys.version_info)) - if py_ver not in SpecifierSet(PYTHON_REQUIRES): - raise SystemExit( - f"This version of pip does not support python {py_ver} " - f"(requires {PYTHON_REQUIRES})" - ) - - sys.meta_path.insert(0, PipImportRedirectingFinder()) assert __name__ == "__main__", "Cannot run __pip-runner__.py as a non-main module" -check_python_version() runpy.run_module("pip", run_name="__main__", alter_sys=True) From 9b638ec6dcf3b28c8c57b0e08056ee19177f52fd Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 3 Aug 2022 20:25:40 +0100 Subject: [PATCH 17/17] Update docs to match behaviour --- docs/html/topics/python-option.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/html/topics/python-option.md b/docs/html/topics/python-option.md index 435d8ed6774..5ad46e7af9c 100644 --- a/docs/html/topics/python-option.md +++ b/docs/html/topics/python-option.md @@ -5,14 +5,13 @@ Occasionally, you may want to use pip to manage a Python installation other than the one pip is installed into. In this case, you can use the `--python` option -to specify the interpreter you want to manage. This option can take one of three +to specify the interpreter you want to manage. This option can take one of two values: 1. The path to a Python executable. 2. The path to a virtual environment. -3. Either "py" or "python", referring to the currently active Python interpreter. -In all 3 cases, pip will run exactly as if it had been invoked from that Python +In both cases, pip will run exactly as if it had been invoked from that Python environment. One example of where this might be useful is to manage a virtual environment