From 3c3fb4152e28d7c8fe113514918378f015497f76 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 31 Jan 2022 16:19:07 -0500 Subject: [PATCH 01/14] pip_api: initial support for hashed requirements --- pip_api/_parse_requirements.py | 34 +++++++++++++++++++++++++++----- tests/test_parse_requirements.py | 12 +++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/pip_api/_parse_requirements.py b/pip_api/_parse_requirements.py index 9a9ea61..e36ce16 100644 --- a/pip_api/_parse_requirements.py +++ b/pip_api/_parse_requirements.py @@ -7,6 +7,8 @@ import string import sys +from collections import defaultdict + from typing import Any, Dict, Optional, Union, Tuple from urllib.parse import urljoin, unquote, urlsplit @@ -24,6 +26,7 @@ parser.add_argument("-i", "--index-url") parser.add_argument("--extra-index-url") parser.add_argument("-f", "--find-links") +parser.add_argument("--hash", action="append", dest="hashes") operators = specifiers.Specifier._operators.keys() @@ -37,7 +40,8 @@ re.VERBOSE, ) WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt") - +# https://pip.pypa.io/en/stable/cli/pip_hash/ +VALID_HASHES = {"sha256", "sha384", "sha512"} class Link: def __init__(self, url): @@ -172,6 +176,14 @@ def _url_to_path(url): return path +class Requirement(requirements.Requirement): + def __init__(self, *args, **kwargs): + self.hashes = kwargs.pop("hashes", None) + + super().__init__(*args, **kwargs) + + + class UnparsedRequirement(object): def __init__(self, name, msg, filename, lineno): self.name = name @@ -446,7 +458,7 @@ def _parse_requirement_url(req_str): def parse_requirements( filename: os.PathLike, options: Optional[Any] = None, include_invalid: bool = False -) -> Dict[str, Union[requirements.Requirement, UnparsedRequirement]]: +) -> Dict[str, Union[Requirement, UnparsedRequirement]]: to_parse = {filename} parsed = set() name_to_req = {} @@ -463,8 +475,20 @@ def parse_requirements( lines_enum = _skip_regex(lines_enum, options) for lineno, line in lines_enum: - req: Optional[Union[requirements.Requirement, UnparsedRequirement]] = None + req: Optional[Union[Requirement, UnparsedRequirement]] = None known, _ = parser.parse_known_args(line.strip().split()) + + hashes_by_kind = defaultdict(list) + if known.hashes: + for hsh in known.hashes: + kind, hsh = hsh.split(":", 1) + if kind not in VALID_HASHES: + raise PipError( + "invalid --hash kind %s, expected one of %s" + % (kind, VALID_HASHES) + ) + hashes_by_kind[kind].append(hsh) + if known.req: req_str = str().join(known.req) try: @@ -477,7 +501,7 @@ def parse_requirements( try: # Try to parse this as a requirement specification if req is None: - req = requirements.Requirement(parsed_req_str) + req = Requirement(parsed_req_str, hashes=dict(hashes_by_kind)) except requirements.InvalidRequirement: try: _check_invalid_requirement(req_str) @@ -493,7 +517,7 @@ def parse_requirements( to_parse.add(full_path) elif known.editable: name, url = _parse_editable(known.editable) - req = requirements.Requirement("%s @ %s" % (name, url)) + req = Requirement("%s @ %s" % (name, url)) else: pass # This is an invalid requirement diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index 77830ab..b1a3b2a 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -253,3 +253,15 @@ def test_parse_requirements_with_missing_egg_suffix(monkeypatch): PipError, match=r"Missing egg fragment in URL: " + PEP508_PIP_EXAMPLE_URL ): pip_api.parse_requirements("a.txt") + + +def test_parse_requirements_hashes(monkeypatch): + files = { + "a.txt": ["foo==1.2.3 --hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e"] + } + monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) + + result = pip_api.parse_requirements("a.txt") + + assert set(result) == {"foo"} + assert result["foo"].hashes == {"sha256": ["862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e"]} From 88386ab5e5b79f1d396392f0a59f38819f7a5c88 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 31 Jan 2022 16:35:01 -0500 Subject: [PATCH 02/14] tests: more tests --- tests/test_parse_requirements.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index b1a3b2a..b39cefa 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -257,11 +257,35 @@ def test_parse_requirements_with_missing_egg_suffix(monkeypatch): def test_parse_requirements_hashes(monkeypatch): files = { - "a.txt": ["foo==1.2.3 --hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e"] + "a.txt": [ + "foo==1.2.3 " + "--hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e " + "--hash=sha256:0cfea7e5a53d5a256b4e8609c8a1812ad9af5c611432ec9dccbb4d79dc6a336e " + "--hash=sha384:673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56 " + "--hash=sha512:3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47" + + ] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") assert set(result) == {"foo"} - assert result["foo"].hashes == {"sha256": ["862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e"]} + assert result["foo"].hashes == { + "sha256": [ + "862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e", + "0cfea7e5a53d5a256b4e8609c8a1812ad9af5c611432ec9dccbb4d79dc6a336e", + ], + "sha384": ["673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56"], + "sha512": ["3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47"], + } + + +def test_parse_requirements_invalid_hash_kind(monkeypatch): + files = { + "a.txt": ["foo==1.2.3 --hash=md5:0d5a28f01dccb5a549c31016883f59c2"] + } + monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) + + with pytest.raises(PipError, match=r"invalid --hash kind"): + pip_api.parse_requirements("a.txt") From 7414cfd413b4db217d21a3ae5f9f7e0962ac60ff Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 31 Jan 2022 16:36:40 -0500 Subject: [PATCH 03/14] pip_api, tests: blacken --- pip_api/_parse_requirements.py | 2 +- tests/test_parse_requirements.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pip_api/_parse_requirements.py b/pip_api/_parse_requirements.py index e36ce16..6240108 100644 --- a/pip_api/_parse_requirements.py +++ b/pip_api/_parse_requirements.py @@ -43,6 +43,7 @@ # https://pip.pypa.io/en/stable/cli/pip_hash/ VALID_HASHES = {"sha256", "sha384", "sha512"} + class Link: def __init__(self, url): # url can be a UNC windows share @@ -183,7 +184,6 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - class UnparsedRequirement(object): def __init__(self, name, msg, filename, lineno): self.name = name diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index b39cefa..ac0af07 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -263,7 +263,6 @@ def test_parse_requirements_hashes(monkeypatch): "--hash=sha256:0cfea7e5a53d5a256b4e8609c8a1812ad9af5c611432ec9dccbb4d79dc6a336e " "--hash=sha384:673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56 " "--hash=sha512:3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47" - ] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) @@ -276,15 +275,17 @@ def test_parse_requirements_hashes(monkeypatch): "862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e", "0cfea7e5a53d5a256b4e8609c8a1812ad9af5c611432ec9dccbb4d79dc6a336e", ], - "sha384": ["673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56"], - "sha512": ["3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47"], + "sha384": [ + "673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56" + ], + "sha512": [ + "3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47" + ], } def test_parse_requirements_invalid_hash_kind(monkeypatch): - files = { - "a.txt": ["foo==1.2.3 --hash=md5:0d5a28f01dccb5a549c31016883f59c2"] - } + files = {"a.txt": ["foo==1.2.3 --hash=md5:0d5a28f01dccb5a549c31016883f59c2"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) with pytest.raises(PipError, match=r"invalid --hash kind"): From 1b779a99c0a6cd759a88e24b465b0bd6b34c3ece Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 31 Jan 2022 16:42:46 -0500 Subject: [PATCH 04/14] tests: add a no-op test for --hash --- tests/test_parse_requirements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index ac0af07..f84055c 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -14,6 +14,7 @@ def test_parse_requirements(monkeypatch): assert set(result) == {"foo"} assert str(result["foo"]) == "foo==1.2.3" + assert result["foo"].hashes == {} def test_parse_requirements_with_comments(monkeypatch): From 621cfc10f9278ed8882af8efc19b4b9e02be4920 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Feb 2022 10:45:14 -0500 Subject: [PATCH 05/14] pip_api, tests: enforce hash strictness --- pip_api/_parse_requirements.py | 17 +++++++++++++++++ tests/test_parse_requirements.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/pip_api/_parse_requirements.py b/pip_api/_parse_requirements.py index 6240108..8b08547 100644 --- a/pip_api/_parse_requirements.py +++ b/pip_api/_parse_requirements.py @@ -462,6 +462,7 @@ def parse_requirements( to_parse = {filename} parsed = set() name_to_req = {} + require_hashes = False while to_parse: filename = to_parse.pop() @@ -478,8 +479,24 @@ def parse_requirements( req: Optional[Union[Requirement, UnparsedRequirement]] = None known, _ = parser.parse_known_args(line.strip().split()) + # If a requirement is missing hashes but we require them, fail. + if not known.hashes and require_hashes: + raise PipError( + "invalid: missing hashes for requirement in %s, line %s" + % (filename, lineno) + ) + + # Similarly, fail if a requirement has hashes but every requirement + # we've parsed previously hasn't had them. + if known.hashes and not require_hashes and len(name_to_req) > 0: + raise PipError( + "invalid: missing hashes for requirements prior to %s, line %s" + % (filename, lineno) + ) + hashes_by_kind = defaultdict(list) if known.hashes: + require_hashes = True for hsh in known.hashes: kind, hsh = hsh.split(":", 1) if kind not in VALID_HASHES: diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index f84055c..db0a0d6 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -291,3 +291,34 @@ def test_parse_requirements_invalid_hash_kind(monkeypatch): with pytest.raises(PipError, match=r"invalid --hash kind"): pip_api.parse_requirements("a.txt") + + +def test_parse_requirements_missing_hashes(monkeypatch): + files = { + "a.txt": [ + "foo==1.2.3 --hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e\n", + "bar==1.2.3\n", + ] + } + monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) + + with pytest.raises( + PipError, match=r"missing hashes for requirement in a\.txt, line 2" + ): + pip_api.parse_requirements("a.txt") + + +def test_parse_requirements_missing_hashes_late(monkeypatch): + files = { + "a.txt": [ + "foo==1.2.3\n", + "bar==1.2.3\n", + "baz==1.2.3 --hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e\n", + ] + } + monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) + + with pytest.raises( + PipError, match=r"missing hashes for requirements prior to a\.txt, line 3" + ): + pip_api.parse_requirements("a.txt") From d967d32c3e6dd00acac30ac52b9fb3b6feddf6de Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Feb 2022 11:32:30 -0500 Subject: [PATCH 06/14] pip_api: add a strict_hashes kwarg to parse_requirements --- README.md | 1 + pip_api/_parse_requirements.py | 14 ++++++++++--- tests/test_parse_requirements.py | 34 ++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7a2bb47..d42dce7 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ If the command you are trying to use is not compatible, `pip_api` will raise a > * `Requirement.marker` ([`packaging.markers.Marker`](https://packaging.pypa.io/en/latest/markers/#packaging.markers.Marker)): A `Marker` of the marker for the requirement. Can be `None`. > Optionally takes an `options` parameter to override the regex used to skip requirements lines. > Optionally takes an `include_invalid` parameter to return an `UnparsedRequirement` in the event that a requirement cannot be parsed correctly. + > Optionally takes a `strict_hashes` parameter to require that all requirements have associated hashes. ### Available with `pip>=8.0.0`: * `pip_api.hash(filename, algorithm='sha256')` diff --git a/pip_api/_parse_requirements.py b/pip_api/_parse_requirements.py index 8b08547..63b751b 100644 --- a/pip_api/_parse_requirements.py +++ b/pip_api/_parse_requirements.py @@ -457,7 +457,10 @@ def _parse_requirement_url(req_str): def parse_requirements( - filename: os.PathLike, options: Optional[Any] = None, include_invalid: bool = False + filename: os.PathLike, + options: Optional[Any] = None, + include_invalid: bool = False, + strict_hashes: bool = False, ) -> Dict[str, Union[Requirement, UnparsedRequirement]]: to_parse = {filename} parsed = set() @@ -480,7 +483,7 @@ def parse_requirements( known, _ = parser.parse_known_args(line.strip().split()) # If a requirement is missing hashes but we require them, fail. - if not known.hashes and require_hashes: + if strict_hashes and not known.hashes and require_hashes: raise PipError( "invalid: missing hashes for requirement in %s, line %s" % (filename, lineno) @@ -488,7 +491,12 @@ def parse_requirements( # Similarly, fail if a requirement has hashes but every requirement # we've parsed previously hasn't had them. - if known.hashes and not require_hashes and len(name_to_req) > 0: + if ( + strict_hashes + and known.hashes + and not require_hashes + and len(name_to_req) > 0 + ): raise PipError( "invalid: missing hashes for requirements prior to %s, line %s" % (filename, lineno) diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index db0a0d6..088d321 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -293,7 +293,11 @@ def test_parse_requirements_invalid_hash_kind(monkeypatch): pip_api.parse_requirements("a.txt") -def test_parse_requirements_missing_hashes(monkeypatch): +@pytest.mark.parametrize( + "strict_hashes", + (True, False), +) +def test_parse_requirements_missing_hashes(monkeypatch, strict_hashes): files = { "a.txt": [ "foo==1.2.3 --hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e\n", @@ -302,13 +306,20 @@ def test_parse_requirements_missing_hashes(monkeypatch): } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) - with pytest.raises( - PipError, match=r"missing hashes for requirement in a\.txt, line 2" - ): - pip_api.parse_requirements("a.txt") + if strict_hashes: + with pytest.raises( + PipError, match=r"missing hashes for requirement in a\.txt, line 2" + ): + pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) + else: + pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) -def test_parse_requirements_missing_hashes_late(monkeypatch): +@pytest.mark.parametrize( + "strict_hashes", + (True, False), +) +def test_parse_requirements_missing_hashes_late(monkeypatch, strict_hashes): files = { "a.txt": [ "foo==1.2.3\n", @@ -318,7 +329,10 @@ def test_parse_requirements_missing_hashes_late(monkeypatch): } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) - with pytest.raises( - PipError, match=r"missing hashes for requirements prior to a\.txt, line 3" - ): - pip_api.parse_requirements("a.txt") + if strict_hashes: + with pytest.raises( + PipError, match=r"missing hashes for requirements prior to a\.txt, line 3" + ): + pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) + else: + pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) From aae9ea0c9549c21dcd8e85d5b725f828e6224c73 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Feb 2022 11:50:10 -0500 Subject: [PATCH 07/14] README: update API line --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d42dce7..9ebd5c1 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ If the command you are trying to use is not compatible, `pip_api` will raise a > * `Distribution.editable` (`bool`): Whether the distribution is editable or not > Optionally takes a `local` parameter to filter out globally-installed packages -* `pip_api.parse_requirements(filename, options=None, include_invalid=False)` +* `pip_api.parse_requirements(filename, options=None, include_invalid=False, strict_hashes=False)` > Takes a path to a filename of a Requirements file. Returns a mapping from package name to a [`packaging.requirements.Requirement`](https://packaging.pypa.io/en/latest/requirements/#packaging.requirements.Requirement) object with the following attributes: > * `Requirement.name` (`string`): The name of the requirement. > * `Requirement.extras` (`set`): A set of extras that the requirement specifies. From d73e0dea4189ec8d944e1cc0e98e93e44b41d6d9 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Feb 2022 11:53:45 -0500 Subject: [PATCH 08/14] tests: add non-strict asserts --- tests/test_parse_requirements.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index 088d321..7c87a95 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -312,7 +312,14 @@ def test_parse_requirements_missing_hashes(monkeypatch, strict_hashes): ): pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) else: - pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) + result = pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) + + assert result["foo"].hashes == { + "sha256": [ + "862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e" + ], + } + assert result["bar"].hashes == {} @pytest.mark.parametrize( @@ -335,4 +342,12 @@ def test_parse_requirements_missing_hashes_late(monkeypatch, strict_hashes): ): pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) else: - pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) + result = pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) + + assert result["foo"].hashes == {} + assert result["bar"].hashes == {} + assert result["baz"].hashes == { + "sha256": [ + "862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e" + ], + } From ebd416d9ff8b7ec8c51d8ff545fe1b810894ba40 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Feb 2022 12:23:10 -0500 Subject: [PATCH 09/14] pip_api: simplify strict hash handling --- pip_api/_parse_requirements.py | 45 ++++++++++++++------------------ tests/test_parse_requirements.py | 6 ++--- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/pip_api/_parse_requirements.py b/pip_api/_parse_requirements.py index 63b751b..d015bdd 100644 --- a/pip_api/_parse_requirements.py +++ b/pip_api/_parse_requirements.py @@ -180,6 +180,8 @@ def _url_to_path(url): class Requirement(requirements.Requirement): def __init__(self, *args, **kwargs): self.hashes = kwargs.pop("hashes", None) + self.filename = kwargs.pop("filename") + self.lineno = kwargs.pop("lineno") super().__init__(*args, **kwargs) @@ -465,7 +467,6 @@ def parse_requirements( to_parse = {filename} parsed = set() name_to_req = {} - require_hashes = False while to_parse: filename = to_parse.pop() @@ -482,34 +483,13 @@ def parse_requirements( req: Optional[Union[Requirement, UnparsedRequirement]] = None known, _ = parser.parse_known_args(line.strip().split()) - # If a requirement is missing hashes but we require them, fail. - if strict_hashes and not known.hashes and require_hashes: - raise PipError( - "invalid: missing hashes for requirement in %s, line %s" - % (filename, lineno) - ) - - # Similarly, fail if a requirement has hashes but every requirement - # we've parsed previously hasn't had them. - if ( - strict_hashes - and known.hashes - and not require_hashes - and len(name_to_req) > 0 - ): - raise PipError( - "invalid: missing hashes for requirements prior to %s, line %s" - % (filename, lineno) - ) - hashes_by_kind = defaultdict(list) if known.hashes: - require_hashes = True for hsh in known.hashes: kind, hsh = hsh.split(":", 1) if kind not in VALID_HASHES: raise PipError( - "invalid --hash kind %s, expected one of %s" + "Invalid --hash kind %s, expected one of %s" % (kind, VALID_HASHES) ) hashes_by_kind[kind].append(hsh) @@ -526,7 +506,12 @@ def parse_requirements( try: # Try to parse this as a requirement specification if req is None: - req = Requirement(parsed_req_str, hashes=dict(hashes_by_kind)) + req = Requirement( + parsed_req_str, + hashes=dict(hashes_by_kind), + filename=filename, + lineno=lineno, + ) except requirements.InvalidRequirement: try: _check_invalid_requirement(req_str) @@ -542,7 +527,9 @@ def parse_requirements( to_parse.add(full_path) elif known.editable: name, url = _parse_editable(known.editable) - req = Requirement("%s @ %s" % (name, url)) + req = Requirement( + "%s @ %s" % (name, url), filename=filename, lineno=lineno + ) else: pass # This is an invalid requirement @@ -561,4 +548,12 @@ def parse_requirements( % (req, name_to_req[req.name], req.name) ) + if strict_hashes: + missing_hashes = [req for req in name_to_req.values() if not req.hashes] + if 0 < len(missing_hashes) < len(name_to_req): + raise PipError( + "Missing hashes for requirement in %s, line %s" + % (missing_hashes[0].filename, missing_hashes[0].lineno) + ) + return name_to_req diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index 7c87a95..4ba97fa 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -289,7 +289,7 @@ def test_parse_requirements_invalid_hash_kind(monkeypatch): files = {"a.txt": ["foo==1.2.3 --hash=md5:0d5a28f01dccb5a549c31016883f59c2"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) - with pytest.raises(PipError, match=r"invalid --hash kind"): + with pytest.raises(PipError, match=r"Invalid --hash kind"): pip_api.parse_requirements("a.txt") @@ -308,7 +308,7 @@ def test_parse_requirements_missing_hashes(monkeypatch, strict_hashes): if strict_hashes: with pytest.raises( - PipError, match=r"missing hashes for requirement in a\.txt, line 2" + PipError, match=r"Missing hashes for requirement in a\.txt, line 2" ): pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) else: @@ -338,7 +338,7 @@ def test_parse_requirements_missing_hashes_late(monkeypatch, strict_hashes): if strict_hashes: with pytest.raises( - PipError, match=r"missing hashes for requirements prior to a\.txt, line 3" + PipError, match=r"Missing hashes for requirement in a\.txt, line 1" ): pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) else: From 7b43b110f02cd474f6bbab402b795ea5039713eb Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Feb 2022 12:33:21 -0500 Subject: [PATCH 10/14] pip_api: make strict_hashes even stricter --- pip_api/_parse_requirements.py | 2 +- tests/test_parse_requirements.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pip_api/_parse_requirements.py b/pip_api/_parse_requirements.py index d015bdd..a23d24d 100644 --- a/pip_api/_parse_requirements.py +++ b/pip_api/_parse_requirements.py @@ -550,7 +550,7 @@ def parse_requirements( if strict_hashes: missing_hashes = [req for req in name_to_req.values() if not req.hashes] - if 0 < len(missing_hashes) < len(name_to_req): + if len(missing_hashes) > 0: raise PipError( "Missing hashes for requirement in %s, line %s" % (missing_hashes[0].filename, missing_hashes[0].lineno) diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py index 4ba97fa..d915fd9 100644 --- a/tests/test_parse_requirements.py +++ b/tests/test_parse_requirements.py @@ -351,3 +351,20 @@ def test_parse_requirements_missing_hashes_late(monkeypatch, strict_hashes): "862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e" ], } + + +def test_parse_requirements_missing_all_hashes_strict(monkeypatch): + files = { + "a.txt": [ + "foo==1.2.3\n", + "bar==1.2.3\n", + "baz==1.2.3\n", + ] + } + + monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) + + with pytest.raises( + PipError, match=r"Missing hashes for requirement in a\.txt, line 1" + ): + pip_api.parse_requirements("a.txt", strict_hashes=True) From 1ef592fbee180b25bba2058d7e7ff431d9fbef4b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Feb 2022 19:47:25 -0500 Subject: [PATCH 11/14] README: improve docs --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9ebd5c1..4f6fe69 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,13 @@ If the command you are trying to use is not compatible, `pip_api` will raise a > Optionally takes a `local` parameter to filter out globally-installed packages * `pip_api.parse_requirements(filename, options=None, include_invalid=False, strict_hashes=False)` - > Takes a path to a filename of a Requirements file. Returns a mapping from package name to a [`packaging.requirements.Requirement`](https://packaging.pypa.io/en/latest/requirements/#packaging.requirements.Requirement) object with the following attributes: + > Takes a path to a filename of a Requirements file. Returns a mapping from package name to a [`packaging.requirements.Requirement`](https://packaging.pypa.io/en/latest/requirements/#packaging.requirements.Requirement) subclass object with the following attributes: > * `Requirement.name` (`string`): The name of the requirement. > * `Requirement.extras` (`set`): A set of extras that the requirement specifies. > * `Requirement.specifier` ([`packaging.specifiers.SpecifierSet`](https://packaging.pypa.io/en/latest/specifiers/#packaging.specifiers.SpecifierSet)): A `SpecifierSet` of the version specified by the requirement. > * `Requirement.marker` ([`packaging.markers.Marker`](https://packaging.pypa.io/en/latest/markers/#packaging.markers.Marker)): A `Marker` of the marker for the requirement. Can be `None`. + > * Additional custom fields: `hashes` for any `--hash=...` options, `filename` and `lineno` for the requirement's parsed location + > > Optionally takes an `options` parameter to override the regex used to skip requirements lines. > Optionally takes an `include_invalid` parameter to return an `UnparsedRequirement` in the event that a requirement cannot be parsed correctly. > Optionally takes a `strict_hashes` parameter to require that all requirements have associated hashes. From a5aaa4d071718e8ababe557bf8370ae0cf61d8bd Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Feb 2022 21:05:40 -0500 Subject: [PATCH 12/14] pip_api: forward Requirement and UnparsedRequirement --- pip_api/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pip_api/__init__.py b/pip_api/__init__.py index c4d9c22..28e3372 100644 --- a/pip_api/__init__.py +++ b/pip_api/__init__.py @@ -14,4 +14,8 @@ from pip_api._installed_distributions import installed_distributions # Import these whenever, doesn't matter -from pip_api._parse_requirements import parse_requirements +from pip_api._parse_requirements import ( + Requirement, + UnparsedRequirement, + parse_requirements, +) From 1f52a90bc7d95af3be7c685e01dc391f934efedd Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 3 Feb 2022 15:35:51 -0500 Subject: [PATCH 13/14] Update README.md Co-authored-by: Dustin Ingram --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f6fe69..9d459cb 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ If the command you are trying to use is not compatible, `pip_api` will raise a > Optionally takes a `local` parameter to filter out globally-installed packages * `pip_api.parse_requirements(filename, options=None, include_invalid=False, strict_hashes=False)` - > Takes a path to a filename of a Requirements file. Returns a mapping from package name to a [`packaging.requirements.Requirement`](https://packaging.pypa.io/en/latest/requirements/#packaging.requirements.Requirement) subclass object with the following attributes: + > Takes a path to a filename of a Requirements file. Returns a mapping from package name to a [`pip_api.Requirement`] object (subclass of [`packaging.requirements.Requirement`](https://packaging.pypa.io/en/latest/requirements/#packaging.requirements.Requirement)) with the following attributes: > * `Requirement.name` (`string`): The name of the requirement. > * `Requirement.extras` (`set`): A set of extras that the requirement specifies. > * `Requirement.specifier` ([`packaging.specifiers.SpecifierSet`](https://packaging.pypa.io/en/latest/specifiers/#packaging.specifiers.SpecifierSet)): A `SpecifierSet` of the version specified by the requirement. From 0d1984f8b1d68177ea2d8ec114e7e3ce96dc84e5 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 3 Feb 2022 15:38:38 -0500 Subject: [PATCH 14/14] README: document all Requirement fields the same --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9d459cb..dcdc4b5 100644 --- a/README.md +++ b/README.md @@ -73,12 +73,14 @@ If the command you are trying to use is not compatible, `pip_api` will raise a > Optionally takes a `local` parameter to filter out globally-installed packages * `pip_api.parse_requirements(filename, options=None, include_invalid=False, strict_hashes=False)` - > Takes a path to a filename of a Requirements file. Returns a mapping from package name to a [`pip_api.Requirement`] object (subclass of [`packaging.requirements.Requirement`](https://packaging.pypa.io/en/latest/requirements/#packaging.requirements.Requirement)) with the following attributes: + > Takes a path to a filename of a Requirements file. Returns a mapping from package name to a `pip_api.Requirement` object (subclass of [`packaging.requirements.Requirement`](https://packaging.pypa.io/en/latest/requirements/#packaging.requirements.Requirement)) with the following attributes: > * `Requirement.name` (`string`): The name of the requirement. > * `Requirement.extras` (`set`): A set of extras that the requirement specifies. > * `Requirement.specifier` ([`packaging.specifiers.SpecifierSet`](https://packaging.pypa.io/en/latest/specifiers/#packaging.specifiers.SpecifierSet)): A `SpecifierSet` of the version specified by the requirement. > * `Requirement.marker` ([`packaging.markers.Marker`](https://packaging.pypa.io/en/latest/markers/#packaging.markers.Marker)): A `Marker` of the marker for the requirement. Can be `None`. - > * Additional custom fields: `hashes` for any `--hash=...` options, `filename` and `lineno` for the requirement's parsed location + > * `Requirement.hashes` (`dict`): A mapping of hashes for the requirement, corresponding to `--hash=...` options. + > * `Requirement.filename` (`str`): The filename that the requirement originates from. + > * `Requirement.lineno` (`int`): The source line that the requirement was parsed from. > > Optionally takes an `options` parameter to override the regex used to skip requirements lines. > Optionally takes an `include_invalid` parameter to return an `UnparsedRequirement` in the event that a requirement cannot be parsed correctly.