From ca6c606e6d5082401affee96dba8f7061276d1cc Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Dec 2022 12:53:36 -0500 Subject: [PATCH 1/2] pip_audit, test: handle invalid requires-python specifiers We follow `pip`'s lead here and ignore these entirely, treating them as if they don't exist (while also warning the user). Fixes #445. See: https://github.com/pypa/packaging/issues/645 Signed-off-by: William Woodruff --- .../resolvelib/pypi_provider.py | 21 +++++++++-- .../resolvelib/test_resolvelib.py | 37 +++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/pip_audit/_dependency_source/resolvelib/pypi_provider.py b/pip_audit/_dependency_source/resolvelib/pypi_provider.py index 484531cd..b3a176e8 100644 --- a/pip_audit/_dependency_source/resolvelib/pypi_provider.py +++ b/pip_audit/_dependency_source/resolvelib/pypi_provider.py @@ -8,6 +8,7 @@ from __future__ import annotations import itertools +import logging from email.message import EmailMessage, Message from email.parser import BytesParser from io import BytesIO @@ -23,7 +24,7 @@ import requests from cachecontrol import CacheControl from packaging.requirements import Requirement -from packaging.specifiers import SpecifierSet +from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import canonicalize_name, parse_sdist_filename, parse_wheel_filename from packaging.version import Version from resolvelib.providers import AbstractProvider @@ -34,6 +35,8 @@ from pip_audit._util import python_version from pip_audit._virtual_env import VirtualEnv, VirtualEnvError +logger = logging.getLogger(__name__) + # TODO: Final[Version] when our minimal Python is 3.8. PYTHON_VERSION: Version = python_version() @@ -246,9 +249,19 @@ def get_project_from_index( py_req = i.attrib.get("data-requires-python") # Skip items that need a different Python version if py_req: - spec = SpecifierSet(py_req) - if PYTHON_VERSION not in spec: - continue + try: + # NOTE: Starting with packaging==22.0, specifier parsing is + # stricter: specifier components can only use the wildcard + # comparison syntax on exact comparison operators (== and !=), + # not on ordered operators like `>=`. There are existing + # packages that use the invalid syntax in their metadata + # however (like nltk==3.6, which does requires-python >= 3.5.*), + # so we follow pip`'s behavior and ignore these specifiers. + spec = SpecifierSet(py_req) + if PYTHON_VERSION not in spec: + continue + except InvalidSpecifier: + logger.warning(f"invalid specifier set for Python version: {py_req}") path = parsed_dist_url.path filename = path.rpartition("/")[-1] diff --git a/test/dependency_source/resolvelib/test_resolvelib.py b/test/dependency_source/resolvelib/test_resolvelib.py index 8f8214b0..286312bb 100644 --- a/test/dependency_source/resolvelib/test_resolvelib.py +++ b/test/dependency_source/resolvelib/test_resolvelib.py @@ -2,6 +2,7 @@ from email.message import EmailMessage +import pretend import pytest import requests from packaging.requirements import Requirement @@ -221,6 +222,42 @@ def test_resolvelib_wheel_python_version(monkeypatch): dict(resolver.resolve_all(iter([req]))) +def test_resolvelib_wheel_python_version_invalid_specifier(monkeypatch): + # requires-python is meant to be a valid specifier version, but earlier + # versions of packaging allowed LegacyVersion parsing for invalid versions. + # This changed in packaging==22.0, so we follow pip's lead and ignore + # any Python version specifiers that aren't valid. + # Note that we intentionally test that version that *should* be skipped + # with a valid specifier (<=3.5.*) is instead included. + data = ( + 'Flask-2.0.1-py3-none-any.whl
' + ) + + logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) + monkeypatch.setattr(pypi_provider, "logger", logger) + + monkeypatch.setattr( + pypi_provider.Candidate, "_get_metadata_for_wheel", lambda _: get_metadata_mock() + ) + + resolver = resolvelib.ResolveLibResolver() + monkeypatch.setattr( + resolver.provider.session, "get", lambda _url, **kwargs: get_package_mock(data) + ) + + req = Requirement("flask==2.0.1") + resolved_deps = dict(resolver.resolve_all(iter([req]))) + assert req in resolved_deps + assert resolved_deps[req] == [ResolvedDependency("flask", Version("2.0.1"))] + + assert logger.warning.calls == [ + pretend.call("invalid specifier set for Python version: <=3.5.*") + ] + + def test_resolvelib_wheel_canonical_name_mismatch(monkeypatch): # Call the underlying wheel, Mask instead of Flask. This should throw an `ResolutionImpossible` # error. From 7e0d3faa27f1557e43e0f619ac3da88c5d69dc9e Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Dec 2022 12:56:43 -0500 Subject: [PATCH 2/2] CHANGELOG: record changes Signed-off-by: William Woodruff --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33207547..182a9b74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ All versions prior to 0.0.9 are untracked. ## [Unreleased] +### Fixed + +* Fixed a crash triggered when a package specifies an invalid version + specifier for its `requires-python` version + ([#447](https://github.com/pypa/pip-audit/pull/447)) + ## [2.4.10] ### Fixed