Skip to content

Commit

Permalink
Merge pull request #10550 from jbylund/joe/cache_requirement_creation
Browse files Browse the repository at this point in the history
  • Loading branch information
pradyunsg authored Oct 9, 2021
2 parents ae2c9ed + 9732729 commit 4fac2b9
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 7 deletions.
1 change: 1 addition & 0 deletions news/10550.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cache requirement objects, to improve performance reducing reparses of requirement strings.
9 changes: 5 additions & 4 deletions src/pip/_internal/req/constructors.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.filetypes import is_archive_file
from pip._internal.utils.misc import is_installable_dir
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.urls import path_to_url
from pip._internal.vcs import is_url, vcs

Expand Down Expand Up @@ -54,7 +55,7 @@ def _strip_extras(path: str) -> Tuple[str, Optional[str]]:
def convert_extras(extras: Optional[str]) -> Set[str]:
if not extras:
return set()
return Requirement("placeholder" + extras.lower()).extras
return get_requirement("placeholder" + extras.lower()).extras


def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
Expand Down Expand Up @@ -83,7 +84,7 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
return (
package_name,
url_no_extras,
Requirement("placeholder" + extras.lower()).extras,
get_requirement("placeholder" + extras.lower()).extras,
)
else:
return package_name, url_no_extras, set()
Expand Down Expand Up @@ -309,7 +310,7 @@ def with_source(text: str) -> str:

def _parse_req_string(req_as_string: str) -> Requirement:
try:
req = Requirement(req_as_string)
req = get_requirement(req_as_string)
except InvalidRequirement:
if os.path.sep in req_as_string:
add_msg = "It looks like a path."
Expand Down Expand Up @@ -386,7 +387,7 @@ def install_req_from_req_string(
user_supplied: bool = False,
) -> InstallRequirement:
try:
req = Requirement(req_string)
req = get_requirement(req_string)
except InvalidRequirement:
raise InstallationError(f"Invalid requirement: '{req_string}'")

Expand Down
4 changes: 2 additions & 2 deletions src/pip/_internal/resolution/resolvelib/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
)

from pip._vendor.packaging.requirements import InvalidRequirement
from pip._vendor.packaging.requirements import Requirement as PackagingRequirement
from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.resolvelib import ResolutionImpossible
Expand All @@ -46,6 +45,7 @@
from pip._internal.resolution.base import InstallRequirementProvider
from pip._internal.utils.compatibility_tags import get_supported
from pip._internal.utils.hashes import Hashes
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.virtualenv import running_under_virtualenv

from .base import Candidate, CandidateVersion, Constraint, Requirement
Expand Down Expand Up @@ -365,7 +365,7 @@ def find_candidates(
# If the current identifier contains extras, add explicit candidates
# from entries from extra-less identifier.
with contextlib.suppress(InvalidRequirement):
parsed_requirement = PackagingRequirement(identifier)
parsed_requirement = get_requirement(identifier)
explicit_candidates.update(
self._iter_explicit_candidates_from_base(
requirements.get(parsed_requirement.name, ()),
Expand Down
13 changes: 13 additions & 0 deletions src/pip/_internal/utils/packaging.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import functools
import logging
from email.message import Message
from email.parser import FeedParser
from typing import Optional, Tuple

from pip._vendor import pkg_resources
from pip._vendor.packaging import specifiers, version
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.pkg_resources import Distribution

from pip._internal.exceptions import NoneMetadataError
Expand Down Expand Up @@ -69,3 +71,14 @@ def get_installer(dist: Distribution) -> str:
if line.strip():
return line.strip()
return ""


@functools.lru_cache(maxsize=512)
def get_requirement(req_string: str) -> Requirement:
"""Construct a packaging.Requirement object with caching"""
# Parsing requirement strings is expensive, and is also expected to happen
# with a low diversity of different arguments (at least relative the number
# constructed). This method adds a cache to requirement object creation to
# minimize repeated parsing of the same string to construct equivalent
# Requirement objects.
return Requirement(req_string)
17 changes: 16 additions & 1 deletion tests/unit/test_packaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@

import pytest
from pip._vendor.packaging import specifiers
from pip._vendor.packaging.requirements import Requirement

from pip._internal.utils.packaging import check_requires_python
from pip._internal.utils.packaging import check_requires_python, get_requirement


@pytest.mark.parametrize(
Expand All @@ -27,3 +28,17 @@ def test_check_requires_python__invalid() -> None:
"""
with pytest.raises(specifiers.InvalidSpecifier):
check_requires_python("invalid", (3, 6, 5))


def test_get_or_create_caching() -> None:
"""test caching of get_or_create requirement"""
teststr = "affinegap==1.10"
from_helper = get_requirement(teststr)
freshly_made = Requirement(teststr)

# Requirement doesn't have an equality operator (yet) so test
# equality of attribute for list of attributes
for iattr in ["name", "url", "extras", "specifier", "marker"]:
assert getattr(from_helper, iattr) == getattr(freshly_made, iattr)
assert get_requirement(teststr) is not Requirement(teststr)
assert get_requirement(teststr) is get_requirement(teststr)

0 comments on commit 4fac2b9

Please sign in to comment.