README, _cli: Add a -S, --strict mode (#146)

* README, _cli: Add a `-S`, `--strict` mode

* README: fix `--help`

* _cli: Use logger.error for _fatal

* _cli: Fix logger.error call

* _cli: Drop the "Fatal: " prefix for logged errors

The logger already provides a reasonable prefix.

* README, _cli: Fix a small typo

* README: fix `--help`

* pip_audit, test: use debug for skipped dep messages

* README, _cli: judicious use of `--help` metavars

* README: fix `--help`

README.md

@@ -26,11 +26,9 @@ python -m pip install pip-audit
 
 <!-- @begin-pip-audit-help@ -->
 ```
-usage: pip-audit [-h] [-V] [-l] [-r REQUIREMENTS]
-                 [-f {columns,json,cyclonedx-json,cyclonedx-xml}]
-                 [-s {osv,pypi}] [-d] [--desc {on,off,auto}]
-                 [--cache-dir CACHE_DIR] [--progress-spinner {on,off}]
-                 [--timeout TIMEOUT]
+usage: pip-audit [-h] [-V] [-l] [-r REQUIREMENTS] [-f FORMAT] [-s SERVICE]
+                 [-d] [-S] [--desc {on,off,auto}] [--cache-dir CACHE_DIR]
+                 [--progress-spinner {on,off}] [--timeout TIMEOUT]
 
 audit the Python environment for dependencies with known vulnerabilities
 
@@ -42,13 +40,17 @@ optional arguments:
   -r REQUIREMENTS, --requirement REQUIREMENTS
                         audit the given requirements file; this option can be
                         used multiple times (default: None)
-  -f {columns,json,cyclonedx-json,cyclonedx-xml}, --format {columns,json,cyclonedx-json,cyclonedx-xml}
-                        the format to emit audit results in (default: columns)
-  -s {osv,pypi}, --vulnerability-service {osv,pypi}
+  -f FORMAT, --format FORMAT
+                        the format to emit audit results in (choices: columns,
+                        json, cyclonedx-json, cyclonedx-xml) (default:
+                        columns)
+  -s SERVICE, --vulnerability-service SERVICE
                         the vulnerability service to audit dependencies
-                        against (default: pypi)
+                        against (choices: osv, pypi) (default: pypi)
   -d, --dry-run         collect all dependencies but do not perform the
                         auditing step (default: False)
+  -S, --strict          fail the entire audit if dependency collection fails
+                        on any dependency (default: False)
   --desc {on,off,auto}  include a description for each vulnerability; `auto`
                         defaults to `on` for the `json` format. This flag has
                         no effect on the `cyclonedx-json` or `cyclonedx-xml`

pip_audit/_cli.py

@@ -9,7 +9,7 @@
 import sys
 from contextlib import ExitStack
 from pathlib import Path
-from typing import List, Optional, cast
+from typing import List, NoReturn, Optional, Type, cast
 
 from pip_audit import __version__
 from pip_audit._audit import AuditOptions, Auditor
@@ -117,6 +117,23 @@ def __str__(self):
         return self.value
 
 
+def _enum_help(msg: str, e: Type[enum.Enum]) -> str:
+    """
+    Render a `--help`-style string for the given enumeration.
+    """
+    return f"{msg} (choices: {', '.join(str(v) for v in e)})"
+
+
+def _fatal(msg: str) -> NoReturn:
+    """
+    Log a fatal error to the standard error stream and exit.
+    """
+    # NOTE: We buffer the logger when the progress spinner is active,
+    # ensuring that the fatal message is formatted on its own line.
+    logger.error(msg)
+    sys.exit(1)
+
+
 def audit() -> None:
     """
     The primary entrypoint for `pip-audit`.
@@ -147,22 +164,32 @@ def audit() -> None:
         type=OutputFormatChoice,
         choices=OutputFormatChoice,
         default=OutputFormatChoice.Columns,
-        help="the format to emit audit results in",
+        metavar="FORMAT",
+        help=_enum_help("the format to emit audit results in", OutputFormatChoice),
     )
     parser.add_argument(
         "-s",
         "--vulnerability-service",
         type=VulnerabilityServiceChoice,
         choices=VulnerabilityServiceChoice,
         default=VulnerabilityServiceChoice.Pypi,
-        help="the vulnerability service to audit dependencies against",
+        metavar="SERVICE",
+        help=_enum_help(
+            "the vulnerability service to audit dependencies against", VulnerabilityServiceChoice
+        ),
     )
     parser.add_argument(
         "-d",
         "--dry-run",
         action="store_true",
         help="collect all dependencies but do not perform the auditing step",
     )
+    parser.add_argument(
+        "-S",
+        "--strict",
+        action="store_true",
+        help="fail the entire audit if dependency collection fails on any dependency",
+    )
     parser.add_argument(
         "--desc",
         type=VulnerabilityDescriptionChoice,
@@ -214,7 +241,10 @@ def audit() -> None:
             if state is not None:
                 if spec.is_skipped():
                     spec = cast(SkippedDependency, spec)
-                    state.update_state(f"Skipping {spec.name}: {spec.skip_reason}")
+                    if args.strict:
+                        _fatal(f"{spec.name}: {spec.skip_reason}")
+                    else:
+                        state.update_state(f"Skipping {spec.name}: {spec.skip_reason}")
                 else:
                     spec = cast(ResolvedDependency, spec)
                     state.update_state(f"Auditing {spec.name} ({spec.version})")

pip_audit/_dependency_source/pip.py

@@ -74,7 +74,7 @@ def collect(self) -> Iterator[Dependency]:
                         "Package has invalid version and could not be audited: "
                         f"{dist.name} ({dist.version})"
                     )
-                    logger.warning(f"Warning: {skip_reason}")
+                    logger.debug(skip_reason)
                     dep = SkippedDependency(name=dist.name, skip_reason=skip_reason)
                 yield dep
         except Exception as e:

pip_audit/_service/pypi.py

@@ -168,7 +168,7 @@ def query(self, spec: Dependency) -> Tuple[Dependency, List[VulnerabilityResult]
                     "Dependency not found on PyPI and could not be audited: "
                     f"{spec.canonical_name} ({spec.version})"
                 )
-                logger.warning(f"Warning: {skip_reason}")
+                logger.debug(skip_reason)
                 return SkippedDependency(name=spec.name, skip_reason=skip_reason), []
             raise ServiceError from http_error
 

test/dependency_source/test_pip.py

@@ -49,7 +49,7 @@ def explode():
 
 
 def test_pip_source_invalid_version(monkeypatch):
-    logger = pretend.stub(warning=pretend.call_recorder(lambda s: None))
+    logger = pretend.stub(debug=pretend.call_recorder(lambda s: None))
     monkeypatch.setattr(pip, "logger", logger)
 
     source = pip.PipSource()
@@ -60,7 +60,7 @@ class MockDistribution:
         version: str
 
     # Return a distribution with a version that doesn't conform to PEP 440.
-    # We should log a warning and skip it.
+    # We should log a debug message and skip it.
     def mock_installed_distributions(local: bool) -> Dict[str, MockDistribution]:
         return {
             "pytest": MockDistribution("pytest", "0.1"),
@@ -71,7 +71,7 @@ def mock_installed_distributions(local: bool) -> Dict[str, MockDistribution]:
     monkeypatch.setattr(pip_api, "installed_distributions", mock_installed_distributions)
 
     specs = list(source.collect())
-    assert len(logger.warning.calls) == 1
+    assert len(logger.debug.calls) == 1
     assert len(specs) == 3
     assert ResolvedDependency(name="pytest", version=Version("0.1")) in specs
     assert (

test/service/test_pypi.py

@@ -49,8 +49,8 @@ def test_pypi_multiple_pkg(cache_dir):
 
 def test_pypi_http_notfound(monkeypatch, cache_dir):
     # If we get a "not found" response, that means that we're querying a package or version that
-    # isn't known to PyPI. If that's the case, we should just log a warning and continue on with
-    # the audit.
+    # isn't known to PyPI. If that's the case, we should just log a debug message and continue on
+    # with the audit.
     def get_error_response():
         class MockResponse:
             # 404: Not Found
@@ -64,7 +64,7 @@ def raise_for_status(self):
     monkeypatch.setattr(
         service.pypi, "_get_cached_session", lambda _: get_mock_session(get_error_response)
     )
-    logger = pretend.stub(warning=pretend.call_recorder(lambda s: None))
+    logger = pretend.stub(debug=pretend.call_recorder(lambda s: None))
     monkeypatch.setattr(service.pypi, "logger", logger)
 
     pypi = service.PyPIService(cache_dir)
@@ -80,7 +80,7 @@ def raise_for_status(self):
     assert skipped_dep in results
     assert dep not in results
     assert len(results[skipped_dep]) == 0
-    assert len(logger.warning.calls) == 1
+    assert len(logger.debug.calls) == 1
 
 
 def test_pypi_http_error(monkeypatch, cache_dir):