From c7dff6ed9954114eb49142a159df2987bd0327c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Eustace?= Date: Fri, 26 Mar 2021 11:29:08 +0100 Subject: [PATCH] Write PEP-610-compliant files when installing --- poetry/installation/executor.py | 135 +++++++++++++++- tests/fixtures/simple_project/pyproject.toml | 5 + tests/installation/test_executor.py | 152 +++++++++++++++++++ 3 files changed, 288 insertions(+), 4 deletions(-) diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index 89ab57a7cf5..506aac8079a 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -2,6 +2,7 @@ from __future__ import division import itertools +import json import os import threading @@ -11,6 +12,7 @@ from subprocess import CalledProcessError from typing import TYPE_CHECKING from typing import Any +from typing import Dict from typing import List from typing import Union @@ -38,6 +40,7 @@ from cleo.io.io import IO # noqa from poetry.config.config import Config + from poetry.core.packages.package import Package from poetry.repositories import Pool from poetry.utils.env import Env @@ -85,6 +88,7 @@ def __init__( self._sections = dict() self._lock = threading.Lock() self._shutdown = False + self._hashes: Dict[str, str] = {} @property def installations_count(self) -> int: @@ -439,10 +443,18 @@ def _display_summary(self, operations: List["OperationTypes"]) -> None: self._io.write_line("") def _execute_install(self, operation: Union[Install, Update]) -> int: - return self._install(operation) + status_code = self._install(operation) + + self._save_url_reference(operation) + + return status_code def _execute_update(self, operation: Union[Install, Update]) -> int: - return self._update(operation) + status_code = self._update(operation) + + self._save_url_reference(operation) + + return status_code def _execute_uninstall(self, operation: Uninstall) -> int: message = ( @@ -599,12 +611,17 @@ def _install_git(self, operation: Union[Install, Update]) -> int: git = Git() git.clone(package.source_url, src_dir) - git.checkout(package.source_reference, src_dir) + git.checkout(package.source_resolved_reference, src_dir) # Now we just need to install from the source directory + original_url = package.source_url package._source_url = str(src_dir) - return self._install_directory(operation) + status_code = self._install_directory(operation) + + package._source_url = original_url + + return status_code def _download(self, operation: Union[Install, Update]) -> Link: link = self._chooser.choose_for(operation.package) @@ -641,6 +658,8 @@ def _download_link(self, operation: Union[Install, Update], link: Link) -> Link: "Invalid hash for {} using archive {}".format(package, archive.name) ) + self._hashes[package.name] = archive_hash + return archive def _download_archive(self, operation: Union[Install, Update], link: Link) -> Path: @@ -694,3 +713,111 @@ def _download_archive(self, operation: Union[Install, Update], link: Link) -> Pa def _should_write_operation(self, operation: Operation) -> bool: return not operation.skipped or self._dry_run or self._verbose + + def _save_url_reference(self, operation: "OperationTypes") -> None: + """ + Create and store a PEP-610 `direct_url.json` file, if needed. + """ + if operation.job_type not in {"install", "update"}: + return + + from poetry.core.masonry.utils.helpers import escape_name + from poetry.core.masonry.utils.helpers import escape_version + + package = operation.package + + if not package.source_url: + # Since we are installing from our own distribution cache + # pip will write a `direct_url.json` file pointing to the cache + # distribution. + # That's not what we want so we remove the direct_url.json file, + # if it exists. + dist_info = self._env.site_packages.path.joinpath( + "{}-{}.dist-info".format( + escape_name(package.pretty_name), + escape_version(package.version.text), + ) + ) + if dist_info.exists() and dist_info.joinpath("direct_url.json").exists(): + dist_info.joinpath("direct_url.json").unlink() + + return + + url_reference = None + + if package.source_type == "git": + url_reference = self._create_git_url_reference(package) + elif package.source_type == "url": + url_reference = self._create_url_url_reference(package) + elif package.source_type == "directory": + url_reference = self._create_directory_url_reference(package) + elif package.source_type == "file": + url_reference = self._create_file_url_reference(package) + + if url_reference: + dist_info = self._env.site_packages.path.joinpath( + "{}-{}.dist-info".format( + escape_name(package.name), escape_version(package.version.text) + ) + ) + + if dist_info.exists(): + dist_info.joinpath("direct_url.json").write_text( + json.dumps(url_reference), encoding="utf-8" + ) + + def _create_git_url_reference( + self, package: "Package" + ) -> Dict[str, Union[str, Dict[str, str]]]: + reference = { + "url": package.source_url, + "vcs_info": { + "vcs": "git", + "requested_revision": package.source_reference, + "commit_id": package.source_resolved_reference, + }, + } + + return reference + + def _create_url_url_reference( + self, package: "Package" + ) -> Dict[str, Union[str, Dict[str, str]]]: + archive_info = {} + + if package.name in self._hashes: + archive_info["hash"] = self._hashes[package.name] + + reference = {"url": package.source_url, "archive_info": archive_info} + + return reference + + def _create_file_url_reference( + self, package: "Package" + ) -> Dict[str, Union[str, Dict[str, str]]]: + archive_info = {} + + if package.name in self._hashes: + archive_info["hash"] = self._hashes[package.name] + + reference = { + "url": Path(package.source_url).as_uri(), + "archive_info": archive_info, + } + + return reference + + def _create_directory_url_reference( + self, package: "Package" + ) -> Dict[str, Union[str, Dict[str, str]]]: + dir_info = {} + + if package.develop: + dir_info["editable"] = True + + reference = { + "url": Path(package.source_url).as_uri(), + "dir_info": dir_info, + } + + return reference diff --git a/tests/fixtures/simple_project/pyproject.toml b/tests/fixtures/simple_project/pyproject.toml index 0fd938e41a0..41a062fc09a 100644 --- a/tests/fixtures/simple_project/pyproject.toml +++ b/tests/fixtures/simple_project/pyproject.toml @@ -28,3 +28,8 @@ python = "~2.7 || ^3.4" foo = "foo:bar" baz = "bar:baz.boom.bim" fox = "fuz.foo:bar.baz" + + +[build-system] +requires = ["poetry-core>=1.0.2"] +build-backend = "poetry.core.masonry.api" diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py index 4511e770a9d..8553d79391e 100644 --- a/tests/installation/test_executor.py +++ b/tests/installation/test_executor.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +import json import re import shutil @@ -19,7 +20,9 @@ from poetry.installation.operations import Uninstall from poetry.installation.operations import Update from poetry.repositories.pool import Pool +from poetry.utils.env import EnvManager from poetry.utils.env import MockEnv +from poetry.utils.env import VirtualEnv from tests.repositories.test_pypi_repository import MockRepository @@ -27,9 +30,21 @@ def env(tmp_dir): path = Path(tmp_dir) / ".venv" path.mkdir(parents=True) + return MockEnv(path=path, is_venv=True) +@pytest.fixture +def venv(tmp_dir): + venv_dir = Path(tmp_dir) / ".venv" + + EnvManager.build_venv(venv_dir) + + yield VirtualEnv(venv_dir, venv_dir) + + EnvManager.remove_venv(venv_dir) + + @pytest.fixture() def io(): io = BufferedIO() @@ -261,3 +276,140 @@ def test_executor_should_delete_incomplete_downloads( executor._download(Install(Package("tomlkit", "0.5.3"))) assert not destination_fixture.exists() + + +def test_executor_should_write_pep610_url_references_for_files(venv, pool, config, io): + url = ( + Path(__file__) + .parent.parent.joinpath( + "fixtures/distributions/demo-0.1.0-py2.py3-none-any.whl" + ) + .resolve() + ) + package = Package("demo", "0.1.0", source_type="file", source_url=url.as_posix()) + + executor = Executor(venv, pool, config, io) + executor.execute([Install(package)]) + + dist_info = venv.site_packages.path.joinpath("demo-0.1.0.dist-info") + assert dist_info.exists() + + direct_url_file = dist_info.joinpath("direct_url.json") + + assert direct_url_file.exists() + + url_reference = json.loads(direct_url_file.read_text(encoding="utf-8")) + + assert url_reference == {"archive_info": {}, "url": url.as_uri()} + + +def test_executor_should_write_pep610_url_references_for_directories( + venv, pool, config, io +): + url = Path(__file__).parent.parent.joinpath("fixtures/simple_project").resolve() + package = Package( + "simple-project", "1.2.3", source_type="directory", source_url=url.as_posix() + ) + + executor = Executor(venv, pool, config, io) + executor.execute([Install(package)]) + + dist_info = venv.site_packages.path.joinpath("simple_project-1.2.3.dist-info") + assert dist_info.exists() + + direct_url_file = dist_info.joinpath("direct_url.json") + + assert direct_url_file.exists() + + url_reference = json.loads(direct_url_file.read_text(encoding="utf-8")) + + assert url_reference == {"dir_info": {}, "url": url.as_uri()} + + +def test_executor_should_write_pep610_url_references_for_editable_directories( + venv, pool, config, io +): + url = Path(__file__).parent.parent.joinpath("fixtures/simple_project").resolve() + package = Package( + "simple-project", + "1.2.3", + source_type="directory", + source_url=url.as_posix(), + develop=True, + ) + + executor = Executor(venv, pool, config, io) + executor.execute([Install(package)]) + + dist_info = venv.site_packages.path.joinpath("simple_project-1.2.3.dist-info") + assert dist_info.exists() + + direct_url_file = dist_info.joinpath("direct_url.json") + + assert direct_url_file.exists() + + url_reference = json.loads(direct_url_file.read_text(encoding="utf-8")) + + assert url_reference == {"dir_info": {"editable": True}, "url": url.as_uri()} + + +def test_executor_should_write_pep610_url_references_for_urls( + venv, pool, config, io, mock_file_downloads +): + package = Package( + "demo", + "0.1.0", + source_type="url", + source_url="https://files.pythonhosted.org/demo-0.1.0-py2.py3-none-any.whl", + ) + + executor = Executor(venv, pool, config, io) + executor.execute([Install(package)]) + + dist_info = venv.site_packages.path.joinpath("demo-0.1.0.dist-info") + assert dist_info.exists() + + direct_url_file = dist_info.joinpath("direct_url.json") + + assert direct_url_file.exists() + + url_reference = json.loads(direct_url_file.read_text(encoding="utf-8")) + + assert url_reference == { + "archive_info": {}, + "url": "https://files.pythonhosted.org/demo-0.1.0-py2.py3-none-any.whl", + } + + +def test_executor_should_write_pep610_url_references_for_git( + venv, pool, config, io, mock_file_downloads +): + package = Package( + "demo", + "0.1.2", + source_type="git", + source_reference="master", + source_resolved_reference="123456", + source_url="https://github.com/demo/demo.git", + ) + + executor = Executor(venv, pool, config, io) + executor.execute([Install(package)]) + + dist_info = venv.site_packages.path.joinpath("demo-0.1.2.dist-info") + assert dist_info.exists() + + direct_url_file = dist_info.joinpath("direct_url.json") + + assert direct_url_file.exists() + + url_reference = json.loads(direct_url_file.read_text(encoding="utf-8")) + + assert url_reference == { + "vcs_info": { + "vcs": "git", + "requested_revision": "master", + "commit_id": "123456", + }, + "url": "https://github.com/demo/demo.git", + }