From 22406d48fce732655fc2aa510cb76783774776d1 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue, 22 Sep 2020 01:26:43 -0700 Subject: [PATCH 1/2] download requirements in the download command, outside of the resolver create PartialRequirementDownloadCompleter, and use in wheel, install, and download add NEWS entry rename NEWS entry rename NEWS entry respond to review comments move the partial requirement download completion to the bottom of the prepare_more method --- news/8896.trivial | 1 + src/pip/_internal/network/download.py | 4 +- src/pip/_internal/operations/prepare.py | 64 ++++++++++++++++++++++--- 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 news/8896.trivial diff --git a/news/8896.trivial b/news/8896.trivial new file mode 100644 index 00000000000..3488b8e057a --- /dev/null +++ b/news/8896.trivial @@ -0,0 +1 @@ +Separate the batched *download* of lazily-fetched wheel files from the preparation of regularly-downloaded requirements in ``RequirementPreparer.prepare_linked_requirements_more()``. diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 76896e89970..bad9e961085 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -178,7 +178,7 @@ def __init__( self._progress_bar = progress_bar def __call__(self, links, location): - # type: (Iterable[Link], str) -> Iterable[Tuple[str, Tuple[str, str]]] + # type: (Iterable[Link], str) -> Iterable[Tuple[Link, Tuple[str, str]]] """Download the files given by links into location.""" for link in links: try: @@ -199,4 +199,4 @@ def __call__(self, links, location): for chunk in chunks: content_file.write(chunk) content_type = resp.headers.get('Content-Type', '') - yield link.url, (filepath, content_type) + yield link, (filepath, content_type) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index de017504abe..45d7341eaf3 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -452,6 +452,39 @@ def _fetch_metadata_using_lazy_wheel(self, link): logger.debug('%s does not support range requests', url) return None + def _complete_partial_requirements( + self, + partially_downloaded_reqs, # type: Iterable[InstallRequirement] + parallel_builds=False, # type: bool + ): + # type: (...) -> None + """Download any requirements which were only fetched by metadata.""" + # Download to a temporary directory. These will be copied over as + # needed for downstream 'download', 'wheel', and 'install' commands. + temp_dir = TempDirectory(kind="unpack", globally_managed=True).path + + # Map each link to the requirement that owns it. This allows us to set + # `req.local_file_path` on the appropriate requirement after passing + # all the links at once into BatchDownloader. + links_to_fully_download = {} # type: Dict[Link, InstallRequirement] + for req in partially_downloaded_reqs: + assert req.link + links_to_fully_download[req.link] = req + + batch_download = self._batch_download( + links_to_fully_download.keys(), + temp_dir, + ) + for link, (filepath, _) in batch_download: + logger.debug("Downloading link %s to %s", link, filepath) + req = links_to_fully_download[link] + req.local_file_path = filepath + + # This step is necessary to ensure all lazy wheels are processed + # successfully by the 'download', 'wheel', and 'install' commands. + for req in partially_downloaded_reqs: + self._prepare_linked_requirement(req, parallel_builds) + def prepare_linked_requirement(self, req, parallel_builds=False): # type: (InstallRequirement, bool) -> Distribution """Prepare a requirement to be obtained from req.link.""" @@ -481,15 +514,32 @@ def prepare_linked_requirement(self, req, parallel_builds=False): def prepare_linked_requirements_more(self, reqs, parallel_builds=False): # type: (Iterable[InstallRequirement], bool) -> None - """Prepare a linked requirement more, if needed.""" + """Prepare linked requirements more, if needed.""" reqs = [req for req in reqs if req.needs_more_preparation] - links = [req.link for req in reqs] - - # Let's download to a temporary directory. - tmpdir = TempDirectory(kind="unpack", globally_managed=True).path - self._downloaded.update(self._batch_download(links, tmpdir)) for req in reqs: - self._prepare_linked_requirement(req, parallel_builds) + # Determine if any of these requirements were already downloaded. + download_dir = self._get_download_dir(req.link) + if download_dir is not None: + hashes = self._get_linked_req_hashes(req) + file_path = _check_download_dir(req.link, download_dir, hashes) + if file_path is not None: + self._downloaded[req.link.url] = file_path, None + req.needs_more_preparation = False + + # Prepare requirements we found were already downloaded for some + # reason. The other downloads will be completed separately. + partially_downloaded_reqs = [] # type: List[InstallRequirement] + for req in reqs: + if req.needs_more_preparation: + partially_downloaded_reqs.append(req) + else: + self._prepare_linked_requirement(req, parallel_builds) + + # TODO: separate this part out from RequirementPreparer when the v1 + # resolver can be removed! + self._complete_partial_requirements( + partially_downloaded_reqs, parallel_builds=parallel_builds, + ) def _prepare_linked_requirement(self, req, parallel_builds): # type: (InstallRequirement, bool) -> Distribution From 24ad324aefab90d396cba725c6076731ee707c6b Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri, 9 Oct 2020 00:44:44 -0700 Subject: [PATCH 2/2] fix lint --- news/{8896.trivial => 8896.trivial.rst} | 0 src/pip/_internal/operations/prepare.py | 5 ++--- 2 files changed, 2 insertions(+), 3 deletions(-) rename news/{8896.trivial => 8896.trivial.rst} (100%) diff --git a/news/8896.trivial b/news/8896.trivial.rst similarity index 100% rename from news/8896.trivial rename to news/8896.trivial.rst diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 45d7341eaf3..7e822f863f9 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -518,10 +518,9 @@ def prepare_linked_requirements_more(self, reqs, parallel_builds=False): reqs = [req for req in reqs if req.needs_more_preparation] for req in reqs: # Determine if any of these requirements were already downloaded. - download_dir = self._get_download_dir(req.link) - if download_dir is not None: + if self.download_dir is not None and req.link.is_wheel: hashes = self._get_linked_req_hashes(req) - file_path = _check_download_dir(req.link, download_dir, hashes) + file_path = _check_download_dir(req.link, self.download_dir, hashes) if file_path is not None: self._downloaded[req.link.url] = file_path, None req.needs_more_preparation = False