From e1a53a548b5e79c6da656ec4f204f9ff889438b8 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Thu, 3 Mar 2022 14:57:21 +1100 Subject: [PATCH 01/15] _cli, pyproject: Add support for `pyproject.toml` files --- pip_audit/_cli.py | 16 ++++- pip_audit/_dependency_source/__init__.py | 2 + pip_audit/_dependency_source/pyproject.py | 82 +++++++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 pip_audit/_dependency_source/pyproject.py diff --git a/pip_audit/_cli.py b/pip_audit/_cli.py index e53c3d37..8577638c 100644 --- a/pip_audit/_cli.py +++ b/pip_audit/_cli.py @@ -17,6 +17,7 @@ PYPI_URL, DependencySource, PipSource, + PyProjectSource, RequirementSource, ResolveLibResolver, ) @@ -159,6 +160,12 @@ def _parser() -> argparse.ArgumentParser: dest="requirements", help="audit the given requirements file; this option can be used multiple times", ) + dep_source_args.add_argument( + "-p", + "--pyproject", + type=argparse.FileType("r"), + help="audit the given `pyproject.toml` file", + ) parser.add_argument( "-f", "--format", @@ -301,8 +308,8 @@ def audit() -> None: state = stack.enter_context(AuditState(members=actors)) source: DependencySource + index_urls = [args.index_url] + args.extra_index_urls if args.requirements is not None: - index_urls = [args.index_url] + args.extra_index_urls req_files: List[Path] = [Path(req.name) for req in args.requirements] source = RequirementSource( req_files, @@ -310,6 +317,13 @@ def audit() -> None: args.require_hashes, state, ) + elif args.pyproject is not None: + pyproject_file = Path(args.pyproject.name) + source = PyProjectSource( + pyproject_file, + ResolveLibResolver(index_urls, args.timeout, args.cache_dir, state), + state, + ) else: source = PipSource(local=args.local, paths=args.paths, state=state) diff --git a/pip_audit/_dependency_source/__init__.py b/pip_audit/_dependency_source/__init__.py index cacb099e..90854008 100644 --- a/pip_audit/_dependency_source/__init__.py +++ b/pip_audit/_dependency_source/__init__.py @@ -10,6 +10,7 @@ DependencySourceError, ) from .pip import PipSource, PipSourceError +from .pyproject import PyProjectSource from .requirement import RequirementSource from .resolvelib import PYPI_URL, ResolveLibResolver @@ -22,6 +23,7 @@ "DependencySourceError", "PipSource", "PipSourceError", + "PyProjectSource", "RequirementSource", "ResolveLibResolver", ] diff --git a/pip_audit/_dependency_source/pyproject.py b/pip_audit/_dependency_source/pyproject.py new file mode 100644 index 00000000..44bef2f5 --- /dev/null +++ b/pip_audit/_dependency_source/pyproject.py @@ -0,0 +1,82 @@ +""" +Collect dependencies from `pyproject.toml` files. +""" + +import logging +from pathlib import Path +from typing import Iterator, List, Set, cast + +import toml +from packaging.requirements import Requirement + +from pip_audit._dependency_source import ( + DependencyFixError, + DependencyResolver, + DependencyResolverError, + DependencySource, + DependencySourceError, +) +from pip_audit._fix import ResolvedFixVersion +from pip_audit._service import Dependency +from pip_audit._service.interface import ResolvedDependency, SkippedDependency +from pip_audit._state import AuditState + +logger = logging.getLogger(__name__) + + +class PyProjectSource(DependencySource): + """ + Wraps `pyproject.toml` dependency resolution as a dependency source. + """ + + def __init__( + self, filename: Path, resolver: DependencyResolver, state: AuditState = AuditState() + ) -> None: + self.filename = filename + self.resolver = resolver + self.state = state + + def collect(self) -> Iterator[Dependency]: + collected: Set[Dependency] = set() + with open(self.filename, "r") as f: + pyproject_data = toml.load(f) + if "project" not in pyproject_data: + raise PyProjectSourceError( + f"pyproject file {self.filename} does not contain `project` section" + ) + project = pyproject_data["project"] + if "dependencies" not in project: + # Projects without dependencies aren't an error case + logger.warn(f"pyproject file {self.filename} does not contain `dependencies` list") + return + deps = project["dependencies"] + reqs: List[Requirement] = [Requirement(dep) for dep in deps] + try: + for _, deps in self.resolver.resolve_all(iter(reqs)): + for dep in deps: + # Don't allow duplicate dependencies to be returned + if dep in collected: + continue + + if dep.is_skipped(): # pragma: no cover + dep = cast(SkippedDependency, dep) + self.state.update_state(f"Skipping {dep.name}: {dep.skip_reason}") + else: + dep = cast(ResolvedDependency, dep) + self.state.update_state(f"Collecting {dep.name} ({dep.version})") + + collected.add(dep) + yield dep + except DependencyResolverError as dre: + raise PyProjectSourceError("dependency resolver raised an error") from dre + + def fix(self, fix_version: ResolvedFixVersion) -> None: + raise NotImplementedError + + +class PyProjectSourceError(DependencySourceError): + pass + + +class PyProjectFixError(DependencyFixError): + pass From 48a9070a6fe9b2afd7e5a0f764b915d804f231b1 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Thu, 3 Mar 2022 15:01:26 +1100 Subject: [PATCH 02/15] pyproject: Add docs --- pip_audit/_dependency_source/pyproject.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pip_audit/_dependency_source/pyproject.py b/pip_audit/_dependency_source/pyproject.py index 44bef2f5..40703616 100644 --- a/pip_audit/_dependency_source/pyproject.py +++ b/pip_audit/_dependency_source/pyproject.py @@ -32,11 +32,26 @@ class PyProjectSource(DependencySource): def __init__( self, filename: Path, resolver: DependencyResolver, state: AuditState = AuditState() ) -> None: + """ + Create a new `PyProjectSource`. + + `filename` provides a path to a `pyproject.toml` file + + `resolver` is the `DependencyResolver` to use. + + `state` is an `AuditState` to use for state callbacks. + """ self.filename = filename self.resolver = resolver self.state = state def collect(self) -> Iterator[Dependency]: + """ + Collect all of the dependencies discovered by this `PyProjectSource`. + + Raises a `PyProjectSourceError` on any errors. + """ + collected: Set[Dependency] = set() with open(self.filename, "r") as f: pyproject_data = toml.load(f) @@ -71,12 +86,20 @@ def collect(self) -> Iterator[Dependency]: raise PyProjectSourceError("dependency resolver raised an error") from dre def fix(self, fix_version: ResolvedFixVersion) -> None: + """ + Fixes a dependency version for this `PyProjectSource`. + """ + raise NotImplementedError class PyProjectSourceError(DependencySourceError): + """A `pyproject.toml` specific `DependencySourceError`.""" + pass class PyProjectFixError(DependencyFixError): + """A `pyproject.toml` specific `DependencyFixError`.""" + pass From d2fab08e8ae54de693a0bbe00fadacafe8d429c7 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Thu, 3 Mar 2022 16:38:02 +1100 Subject: [PATCH 03/15] pyproject: Support fixing of `pyproject.toml` --- pip_audit/_dependency_source/pyproject.py | 29 ++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pip_audit/_dependency_source/pyproject.py b/pip_audit/_dependency_source/pyproject.py index 40703616..33cc7c3d 100644 --- a/pip_audit/_dependency_source/pyproject.py +++ b/pip_audit/_dependency_source/pyproject.py @@ -3,11 +3,14 @@ """ import logging +import shutil from pathlib import Path +from tempfile import NamedTemporaryFile from typing import Iterator, List, Set, cast import toml from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet from pip_audit._dependency_source import ( DependencyFixError, @@ -90,7 +93,31 @@ def fix(self, fix_version: ResolvedFixVersion) -> None: Fixes a dependency version for this `PyProjectSource`. """ - raise NotImplementedError + with open(self.filename, "r+") as f, NamedTemporaryFile(mode="r+") as tmp: + pyproject_data = toml.load(f) + if "project" not in pyproject_data: + raise PyProjectFixError( + f"pyproject file {self.filename} does not contain `project` section" + ) + project = pyproject_data["project"] + if "dependencies" not in project: + # Projects without dependencies aren't an error case + logger.warn(f"pyproject file {self.filename} does not contain `dependencies` list") + return + deps = project["dependencies"] + reqs: List[Requirement] = [Requirement(dep) for dep in deps] + for i in range(len(reqs)): + req = reqs[i] + if ( + req.name == fix_version.dep.name + and req.specifier.contains(fix_version.dep.version) + and not req.specifier.contains(fix_version.version) + ): + req.specifier = SpecifierSet(f"=={fix_version.version}") + deps[i] = str(req) + assert req.marker is None or req.marker.evaluate() + toml.dump(pyproject_data, tmp) + shutil.copyfileobj(tmp, f) class PyProjectSourceError(DependencySourceError): From 51050c2565dd800dcf161eec60b178a25af85ee1 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Thu, 3 Mar 2022 16:47:51 +1100 Subject: [PATCH 04/15] pyproject: Use an `os.replace` to move the tmp file into place --- pip_audit/_dependency_source/pyproject.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pip_audit/_dependency_source/pyproject.py b/pip_audit/_dependency_source/pyproject.py index 33cc7c3d..cd4faed1 100644 --- a/pip_audit/_dependency_source/pyproject.py +++ b/pip_audit/_dependency_source/pyproject.py @@ -3,7 +3,7 @@ """ import logging -import shutil +import os from pathlib import Path from tempfile import NamedTemporaryFile from typing import Iterator, List, Set, cast @@ -104,9 +104,12 @@ def fix(self, fix_version: ResolvedFixVersion) -> None: # Projects without dependencies aren't an error case logger.warn(f"pyproject file {self.filename} does not contain `dependencies` list") return + deps = project["dependencies"] reqs: List[Requirement] = [Requirement(dep) for dep in deps] for i in range(len(reqs)): + # When we find a requirement that matches the provided fix version, we need to edit + # the requirement's specifier and then write it back to the underlying TOML data. req = reqs[i] if ( req.name == fix_version.dep.name @@ -116,8 +119,16 @@ def fix(self, fix_version: ResolvedFixVersion) -> None: req.specifier = SpecifierSet(f"=={fix_version.version}") deps[i] = str(req) assert req.marker is None or req.marker.evaluate() + + # Now dump the new edited TOML to the temporary file. toml.dump(pyproject_data, tmp) - shutil.copyfileobj(tmp, f) + + # And replace the original `pyproject.toml` file + os.replace(tmp.name, self.filename) + + # Stop the file wrapper from attempting to cleanup if we've successfully moved the + # temporary file into place. + tmp._closer.delete = False class PyProjectSourceError(DependencySourceError): From 61e1fb1d0230410f7eb04dcaaa96e74b4ea2237a Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 13:39:58 +1100 Subject: [PATCH 05/15] pyproject, test: Add unit tests for PyProject support --- pip_audit/_dependency_source/pyproject.py | 10 ++- test/dependency_source/test_pyproject.py | 79 +++++++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 test/dependency_source/test_pyproject.py diff --git a/pip_audit/_dependency_source/pyproject.py b/pip_audit/_dependency_source/pyproject.py index cd4faed1..43dd6186 100644 --- a/pip_audit/_dependency_source/pyproject.py +++ b/pip_audit/_dependency_source/pyproject.py @@ -65,7 +65,9 @@ def collect(self) -> Iterator[Dependency]: project = pyproject_data["project"] if "dependencies" not in project: # Projects without dependencies aren't an error case - logger.warn(f"pyproject file {self.filename} does not contain `dependencies` list") + logger.warning( + f"pyproject file {self.filename} does not contain `dependencies` list" + ) return deps = project["dependencies"] reqs: List[Requirement] = [Requirement(dep) for dep in deps] @@ -102,7 +104,9 @@ def fix(self, fix_version: ResolvedFixVersion) -> None: project = pyproject_data["project"] if "dependencies" not in project: # Projects without dependencies aren't an error case - logger.warn(f"pyproject file {self.filename} does not contain `dependencies` list") + logger.warning( + f"pyproject file {self.filename} does not contain `dependencies` list" + ) return deps = project["dependencies"] @@ -123,7 +127,7 @@ def fix(self, fix_version: ResolvedFixVersion) -> None: # Now dump the new edited TOML to the temporary file. toml.dump(pyproject_data, tmp) - # And replace the original `pyproject.toml` file + # And replace the original `pyproject.toml` file. os.replace(tmp.name, self.filename) # Stop the file wrapper from attempting to cleanup if we've successfully moved the diff --git a/test/dependency_source/test_pyproject.py b/test/dependency_source/test_pyproject.py new file mode 100644 index 00000000..da2433f0 --- /dev/null +++ b/test/dependency_source/test_pyproject.py @@ -0,0 +1,79 @@ +from pathlib import Path + +import pretend # type: ignore +import pytest +from packaging.version import Version + +from pip_audit._dependency_source import DependencySourceError, ResolveLibResolver, pyproject +from pip_audit._service import ResolvedDependency + + +def __init_pyproject(filename: Path, contents: str) -> Path: + with open(filename, mode="w") as f: + f.write(contents) + return filename + + +@pytest.mark.online +def test_pyproject_source(req_file): + filename = __init_pyproject( + req_file(), + """ +[project] +dependencies = [ + "flask==2.0.1" +] + """, + ) + source = pyproject.PyProjectSource(filename, ResolveLibResolver()) + specs = list(source.collect()) + assert ResolvedDependency("flask", Version("2.0.1")) in specs + + +def test_pyproject_source_no_project_section(req_file): + filename = __init_pyproject( + req_file(), + """ +[some_other_section] +dependencies = [ + "flask==2.0.1" +] + """, + ) + source = pyproject.PyProjectSource(filename, ResolveLibResolver()) + with pytest.raises(DependencySourceError): + list(source.collect()) + + +def test_pyproject_source_no_deps(monkeypatch, req_file): + logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) + monkeypatch.setattr(pyproject, "logger", logger) + + filename = __init_pyproject( + req_file(), + """ +[project] + """, + ) + source = pyproject.PyProjectSource(filename, ResolveLibResolver()) + specs = list(source.collect()) + assert not specs + + # We log a warning when we find a `pyproject.toml` file with no dependencies + assert len(logger.warning.calls) == 1 + + +def test_pyproject_source_duplicate_deps(req_file): + filename = __init_pyproject( + req_file(), + """ +[project] +dependencies = [ + "flask", + "click", +] +""", + ) + source = pyproject.PyProjectSource(filename, ResolveLibResolver()) + specs = list(source.collect()) + assert len(specs) == len(set(specs)) From ecb805bcd5ebac2da633c8bb9dd70b7743afa8a0 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 14:11:52 +1100 Subject: [PATCH 06/15] test: Finish off PyProject source collect tests --- test/dependency_source/test_pyproject.py | 45 ++++++++++++++++++++---- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/test/dependency_source/test_pyproject.py b/test/dependency_source/test_pyproject.py index da2433f0..f7fcc106 100644 --- a/test/dependency_source/test_pyproject.py +++ b/test/dependency_source/test_pyproject.py @@ -1,14 +1,22 @@ from pathlib import Path +from typing import List import pretend # type: ignore import pytest +from packaging.requirements import Requirement from packaging.version import Version -from pip_audit._dependency_source import DependencySourceError, ResolveLibResolver, pyproject -from pip_audit._service import ResolvedDependency +from pip_audit._dependency_source import ( + DependencyResolver, + DependencyResolverError, + DependencySourceError, + ResolveLibResolver, + pyproject, +) +from pip_audit._service import Dependency, ResolvedDependency -def __init_pyproject(filename: Path, contents: str) -> Path: +def _init_pyproject(filename: Path, contents: str) -> Path: with open(filename, mode="w") as f: f.write(contents) return filename @@ -16,7 +24,7 @@ def __init_pyproject(filename: Path, contents: str) -> Path: @pytest.mark.online def test_pyproject_source(req_file): - filename = __init_pyproject( + filename = _init_pyproject( req_file(), """ [project] @@ -31,7 +39,7 @@ def test_pyproject_source(req_file): def test_pyproject_source_no_project_section(req_file): - filename = __init_pyproject( + filename = _init_pyproject( req_file(), """ [some_other_section] @@ -49,7 +57,7 @@ def test_pyproject_source_no_deps(monkeypatch, req_file): logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) monkeypatch.setattr(pyproject, "logger", logger) - filename = __init_pyproject( + filename = _init_pyproject( req_file(), """ [project] @@ -64,7 +72,9 @@ def test_pyproject_source_no_deps(monkeypatch, req_file): def test_pyproject_source_duplicate_deps(req_file): - filename = __init_pyproject( + # Click is a dependency of Flask. We should check that the dependencies of Click aren't returned + # twice. + filename = _init_pyproject( req_file(), """ [project] @@ -76,4 +86,25 @@ def test_pyproject_source_duplicate_deps(req_file): ) source = pyproject.PyProjectSource(filename, ResolveLibResolver()) specs = list(source.collect()) + + # Check that the list of dependencies is already deduplicated assert len(specs) == len(set(specs)) + + +def test_pyproject_source_resolver_error(monkeypatch, req_file): + class MockResolver(DependencyResolver): + def resolve(self, req: Requirement) -> List[Dependency]: + raise DependencyResolverError + + filename = _init_pyproject( + req_file(), + """ +[project] +dependencies = [ + "flask==2.0.1" +] +""", + ) + source = pyproject.PyProjectSource(filename, MockResolver()) + with pytest.raises(DependencySourceError): + list(source.collect()) From 7b772aea213e67fe9835e47042ff6f3e6b15e835 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 18:33:57 +1100 Subject: [PATCH 07/15] _cli: Create positional `project_path` arg --- pip_audit/_cli.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/pip_audit/_cli.py b/pip_audit/_cli.py index 8577638c..039c4764 100644 --- a/pip_audit/_cli.py +++ b/pip_audit/_cli.py @@ -161,10 +161,7 @@ def _parser() -> argparse.ArgumentParser: help="audit the given requirements file; this option can be used multiple times", ) dep_source_args.add_argument( - "-p", - "--pyproject", - type=argparse.FileType("r"), - help="audit the given `pyproject.toml` file", + "project_path", type=Path, nargs="?", help="audit a local Python project at the given path" ) parser.add_argument( "-f", @@ -276,6 +273,15 @@ def _parse_args(parser: argparse.ArgumentParser) -> argparse.Namespace: return parser.parse_args() +def _create_project_source(project_path: Path, state: AuditState) -> Optional[DependencySource]: + # Check for a `pyproject.toml` + pyproject_path = project_path.joinpath("pyproject.toml") + if pyproject_path.is_file(): + return PyProjectSource(pyproject_path, ResolveLibResolver(), state) + else: + return None + + def audit() -> None: """ The primary entrypoint for `pip-audit`. @@ -317,13 +323,12 @@ def audit() -> None: args.require_hashes, state, ) - elif args.pyproject is not None: - pyproject_file = Path(args.pyproject.name) - source = PyProjectSource( - pyproject_file, - ResolveLibResolver(index_urls, args.timeout, args.cache_dir, state), - state, - ) + elif args.project_path is not None: + # Determine which kind of project file exists in the project path + _source = _create_project_source(args.project_path, state) + if _source is None: + parser.error(f"couldn't find project file at path: {args.project_path}") + source = _source else: source = PipSource(local=args.local, paths=args.paths, state=state) From 547aed4b1cece1f86db6536be55ad0c2d160a97c Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 18:36:22 +1100 Subject: [PATCH 08/15] pyproject: Reorganise imports --- pip_audit/_dependency_source/pyproject.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pip_audit/_dependency_source/pyproject.py b/pip_audit/_dependency_source/pyproject.py index 43dd6186..9f65229c 100644 --- a/pip_audit/_dependency_source/pyproject.py +++ b/pip_audit/_dependency_source/pyproject.py @@ -20,8 +20,7 @@ DependencySourceError, ) from pip_audit._fix import ResolvedFixVersion -from pip_audit._service import Dependency -from pip_audit._service.interface import ResolvedDependency, SkippedDependency +from pip_audit._service import Dependency, ResolvedDependency, SkippedDependency from pip_audit._state import AuditState logger = logging.getLogger(__name__) From 02487de11f72991da28e8f97b12d2afb51d79fd5 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 18:52:47 +1100 Subject: [PATCH 09/15] test: Mark duplicate deps test as `online` --- test/dependency_source/test_pyproject.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/dependency_source/test_pyproject.py b/test/dependency_source/test_pyproject.py index f7fcc106..b4091c3b 100644 --- a/test/dependency_source/test_pyproject.py +++ b/test/dependency_source/test_pyproject.py @@ -71,6 +71,7 @@ def test_pyproject_source_no_deps(monkeypatch, req_file): assert len(logger.warning.calls) == 1 +@pytest.mark.online def test_pyproject_source_duplicate_deps(req_file): # Click is a dependency of Flask. We should check that the dependencies of Click aren't returned # twice. From 5e994b22ddcfd7fa80d53c70dcab655afe6866f3 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 19:42:40 +1100 Subject: [PATCH 10/15] test: Add unit tests for fixing PyProject files --- test/dependency_source/test_pyproject.py | 82 ++++++++++++++++++++---- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/test/dependency_source/test_pyproject.py b/test/dependency_source/test_pyproject.py index b4091c3b..f882b2bb 100644 --- a/test/dependency_source/test_pyproject.py +++ b/test/dependency_source/test_pyproject.py @@ -3,6 +3,7 @@ import pretend # type: ignore import pytest +import toml from packaging.requirements import Requirement from packaging.version import Version @@ -13,18 +14,24 @@ ResolveLibResolver, pyproject, ) +from pip_audit._dependency_source.interface import DependencyFixError, ResolvedFixVersion from pip_audit._service import Dependency, ResolvedDependency -def _init_pyproject(filename: Path, contents: str) -> Path: +def _init_pyproject(filename: Path, contents: str) -> pyproject.PyProjectSource: with open(filename, mode="w") as f: f.write(contents) - return filename + return pyproject.PyProjectSource(filename, ResolveLibResolver()) + + +def _check_file(filename: Path, expected_contents: dict) -> None: + with open(filename, mode="r") as f: + assert toml.load(f) == expected_contents @pytest.mark.online def test_pyproject_source(req_file): - filename = _init_pyproject( + source = _init_pyproject( req_file(), """ [project] @@ -33,13 +40,12 @@ def test_pyproject_source(req_file): ] """, ) - source = pyproject.PyProjectSource(filename, ResolveLibResolver()) specs = list(source.collect()) assert ResolvedDependency("flask", Version("2.0.1")) in specs def test_pyproject_source_no_project_section(req_file): - filename = _init_pyproject( + source = _init_pyproject( req_file(), """ [some_other_section] @@ -48,7 +54,6 @@ def test_pyproject_source_no_project_section(req_file): ] """, ) - source = pyproject.PyProjectSource(filename, ResolveLibResolver()) with pytest.raises(DependencySourceError): list(source.collect()) @@ -57,13 +62,12 @@ def test_pyproject_source_no_deps(monkeypatch, req_file): logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) monkeypatch.setattr(pyproject, "logger", logger) - filename = _init_pyproject( + source = _init_pyproject( req_file(), """ [project] """, ) - source = pyproject.PyProjectSource(filename, ResolveLibResolver()) specs = list(source.collect()) assert not specs @@ -75,7 +79,7 @@ def test_pyproject_source_no_deps(monkeypatch, req_file): def test_pyproject_source_duplicate_deps(req_file): # Click is a dependency of Flask. We should check that the dependencies of Click aren't returned # twice. - filename = _init_pyproject( + source = _init_pyproject( req_file(), """ [project] @@ -85,7 +89,6 @@ def test_pyproject_source_duplicate_deps(req_file): ] """, ) - source = pyproject.PyProjectSource(filename, ResolveLibResolver()) specs = list(source.collect()) # Check that the list of dependencies is already deduplicated @@ -97,7 +100,7 @@ class MockResolver(DependencyResolver): def resolve(self, req: Requirement) -> List[Dependency]: raise DependencyResolverError - filename = _init_pyproject( + source = _init_pyproject( req_file(), """ [project] @@ -106,6 +109,61 @@ def resolve(self, req: Requirement) -> List[Dependency]: ] """, ) - source = pyproject.PyProjectSource(filename, MockResolver()) + source.resolver = MockResolver() with pytest.raises(DependencySourceError): list(source.collect()) + + +@pytest.mark.online +def test_pyproject_source_fix(req_file): + source = _init_pyproject( + req_file(), + """ +[project] +dependencies = [ + "flask==0.5" +] +""", + ) + fix = ResolvedFixVersion( + dep=ResolvedDependency(name="flask", version=Version("0.5")), version=Version("1.0") + ) + source.fix(fix) + _check_file(source.filename, {"project": {"dependencies": ["flask==1.0"]}}) + + +def test_pyproject_source_fix_no_project_section(req_file): + source = _init_pyproject( + req_file(), + """ +[some_other_section] +dependencies = [ + "flask==2.0.1" +] +""", + ) + fix = ResolvedFixVersion( + dep=ResolvedDependency(name="flask", version=Version("0.5")), version=Version("1.0") + ) + with pytest.raises(DependencyFixError): + source.fix(fix) + + +def test_pyproject_source_fix_no_deps(monkeypatch, req_file): + logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) + monkeypatch.setattr(pyproject, "logger", logger) + + source = _init_pyproject( + req_file(), + """ +[project] +""", + ) + fix = ResolvedFixVersion( + dep=ResolvedDependency(name="flask", version=Version("0.5")), version=Version("1.0") + ) + source.fix(fix) + + # We log a warning when we find a `pyproject.toml` file with no dependencies + assert len(logger.warning.calls) == 1 + _check_file(source.filename, {"project": {}}) From b218be63263032a450f1d707337a2bb91e50b089 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 19:46:43 +1100 Subject: [PATCH 11/15] README: Update help text --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 16ec7937..edeafd60 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,14 @@ usage: pip-audit [-h] [-V] [-l] [-r REQUIREMENTS] [-f FORMAT] [-s SERVICE] [--progress-spinner {on,off}] [--timeout TIMEOUT] [--path PATHS] [-v] [--fix] [--require-hashes] [--index-url INDEX_URL] [--extra-index-url EXTRA_INDEX_URLS] + [project_path] audit the Python environment for dependencies with known vulnerabilities +positional arguments: + project_path audit a local Python project at the given path + (default: None) + optional arguments: -h, --help show this help message and exit -V, --version show program's version number and exit From 5f4cd0c6dcdd5e4b33dfdd4b5811c868b5fed892 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 19:53:38 +1100 Subject: [PATCH 12/15] README: Fix help text --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 68da334b..beca170a 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,8 @@ usage: pip-audit [-h] [-V] [-l] [-r REQUIREMENTS] [-f FORMAT] [-s SERVICE] [--progress-spinner {on,off}] [--timeout TIMEOUT] [--path PATHS] [-v] [--fix] [--require-hashes] [--index-url INDEX_URL] [--extra-index-url EXTRA_INDEX_URLS] - [--skip-editable] [project_path] + [--skip-editable] + [project_path] audit the Python environment for dependencies with known vulnerabilities From dc47ef7701f4d3160692f3af115a3bd72d8a066a Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Fri, 4 Mar 2022 20:24:09 +1100 Subject: [PATCH 13/15] README: Add example invocation for project files --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index beca170a..d25ed25a 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,13 @@ $ pip-audit -r ./requirements.txt -l No known vulnerabilities found ``` +Audit dependencies for a local Python project: +``` +$ pip-audit . +No known vulnerabilities found +``` +`pip-audit` searches the provided path for various Python "project" files. At the moment, only `pyproject.toml` is supported. + Audit dependencies when there are vulnerabilities present: ``` $ pip-audit From 370210bb650b82d137c56638677090df1a8b6158 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 4 Mar 2022 10:35:20 -0500 Subject: [PATCH 14/15] pip_audit: cleanup, notes --- pip_audit/_cli.py | 18 +++++++------- pip_audit/_dependency_source/pyproject.py | 29 ++++++++++++----------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/pip_audit/_cli.py b/pip_audit/_cli.py index 4c51293b..5c1432b6 100644 --- a/pip_audit/_cli.py +++ b/pip_audit/_cli.py @@ -278,13 +278,15 @@ def _parse_args(parser: argparse.ArgumentParser) -> argparse.Namespace: return parser.parse_args() -def _create_project_source(project_path: Path, state: AuditState) -> Optional[DependencySource]: +def _dep_source_from_project_path(project_path: Path, state: AuditState) -> DependencySource: # Check for a `pyproject.toml` - pyproject_path = project_path.joinpath("pyproject.toml") + pyproject_path = project_path / "pyproject.toml" if pyproject_path.is_file(): return PyProjectSource(pyproject_path, ResolveLibResolver(), state) - else: - return None + + # TODO: Checks for setup.py and other project files will go here. + + _fatal(f"couldn't find a supported project file in {project_path}") def audit() -> None: @@ -333,11 +335,11 @@ def audit() -> None: state=state, ) elif args.project_path is not None: + # NOTE: We'll probably want to support --skip-editable here, + # once PEP 660 is more widely supported: https://www.python.org/dev/peps/pep-0660/ + # Determine which kind of project file exists in the project path - _source = _create_project_source(args.project_path, state) - if _source is None: - parser.error(f"couldn't find project file at path: {args.project_path}") - source = _source + source = _dep_source_from_project_path(args.project_path, state) else: source = PipSource( local=args.local, paths=args.paths, skip_editable=args.skip_editable, state=state diff --git a/pip_audit/_dependency_source/pyproject.py b/pip_audit/_dependency_source/pyproject.py index 9f65229c..b8cec200 100644 --- a/pip_audit/_dependency_source/pyproject.py +++ b/pip_audit/_dependency_source/pyproject.py @@ -55,20 +55,23 @@ def collect(self) -> Iterator[Dependency]: """ collected: Set[Dependency] = set() - with open(self.filename, "r") as f: + with self.filename.open("r") as f: pyproject_data = toml.load(f) - if "project" not in pyproject_data: + + project = pyproject_data.get("project") + if project is None: raise PyProjectSourceError( f"pyproject file {self.filename} does not contain `project` section" ) - project = pyproject_data["project"] - if "dependencies" not in project: + + deps = project.get("dependencies") + if deps is None: # Projects without dependencies aren't an error case logger.warning( f"pyproject file {self.filename} does not contain `dependencies` list" ) return - deps = project["dependencies"] + reqs: List[Requirement] = [Requirement(dep) for dep in deps] try: for _, deps in self.resolver.resolve_all(iter(reqs)): @@ -94,21 +97,23 @@ def fix(self, fix_version: ResolvedFixVersion) -> None: Fixes a dependency version for this `PyProjectSource`. """ - with open(self.filename, "r+") as f, NamedTemporaryFile(mode="r+") as tmp: + with self.filename.open("r+") as f, NamedTemporaryFile(mode="r+", delete=False) as tmp: pyproject_data = toml.load(f) - if "project" not in pyproject_data: + + project = pyproject_data.get("project") + if project is None: raise PyProjectFixError( f"pyproject file {self.filename} does not contain `project` section" ) - project = pyproject_data["project"] - if "dependencies" not in project: + + deps = project.get("dependencies") + if deps is None: # Projects without dependencies aren't an error case logger.warning( f"pyproject file {self.filename} does not contain `dependencies` list" ) return - deps = project["dependencies"] reqs: List[Requirement] = [Requirement(dep) for dep in deps] for i in range(len(reqs)): # When we find a requirement that matches the provided fix version, we need to edit @@ -129,10 +134,6 @@ def fix(self, fix_version: ResolvedFixVersion) -> None: # And replace the original `pyproject.toml` file. os.replace(tmp.name, self.filename) - # Stop the file wrapper from attempting to cleanup if we've successfully moved the - # temporary file into place. - tmp._closer.delete = False - class PyProjectSourceError(DependencySourceError): """A `pyproject.toml` specific `DependencySourceError`.""" From 4891c5ae2ab65ff54a5495075fbaae806d6eb683 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 4 Mar 2022 10:37:41 -0500 Subject: [PATCH 15/15] CHANGELOG: record changes --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9a0d87a..f25ea8f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ All versions prior to 0.0.9 are untracked. packages or parsed requirements (via `-r`) that are marked as editable ([#244](https://github.com/trailofbits/pip-audit/pull/244)) +* CLI: `pip-audit` can audit projects that list their dependencies in + `pyproject.toml` files, via `pip-audit ` + ([#246](https://github.com/trailofbits/pip-audit/pull/246)) + ## [2.0.0] - 2022-02-18 ### Added