Cargo - Skip existing packages when publishing (#280)

* WIP - Working implementation of checking package existence through cargo's sparse API

* Refactor `prepare` to catch potential exceptions

* Fix source

* Wire allow_overwrite in public `cargo_publish` function

* If package / version are not available, pull from manifest

* Move methods for readability

* Add Changelog

* krakenw r fmt lint

* Enable test

* Update kraken-build/src/kraken/std/cargo/tasks/cargo_publish_task.py

Co-authored-by: Niklas Rosenstein <[email protected]>

* CR - nits

* Assert that publish task is skipped

* tmp: add logs to try things out in CI

* Fix usage of get_or that returned a 'none'

* fmt

* Mock the registry to test skipping

* lint

* Apply suggestions from code review

---------

Co-authored-by: Jean Caillé <[email protected]>
Co-authored-by: Niklas Rosenstein <[email protected]>

kraken-build/.changelog/_unreleased.toml

@@ -0,0 +1,5 @@
+[[entries]]
+id = "48a06299-f287-49b8-8c91-1f74cfa56f2f"
+type = "feature"
+description = "Cargo Publishing : Skip publishing packages that already exists in the registry"
+author = "@jcaille"

kraken-build/pyproject.toml

@@ -64,6 +64,7 @@ types-requests = "^2.28.0"
 pytest-xdist = {version = "^3.5.0", extras = ["psutil"]}
 mitmproxy = "^10.2.4"
 types-networkx = "^3.2.1.20240703"
+requests-mock = "^1.12.1"
 
 # Slap configuration
 # ------------------

kraken-build/src/kraken/core/system/context.py

@@ -357,7 +357,7 @@ def get_build_graph(self, targets: Sequence[str | Address | Task] | None) -> Tas
         assert graph, "TaskGraph cannot be empty"
         return graph
 
-    def execute(self, tasks: list[str | Address | Task] | TaskGraph | None = None) -> None:
+    def execute(self, tasks: list[str | Address | Task] | TaskGraph | None = None) -> TaskGraph:
         """Execute all default tasks or the tasks specified by *targets* using the default executor.
         If :meth:`finalize` was not called already it will be called by this function before the build
         graph is created, unless a build graph is passed in the first place.
@@ -379,6 +379,7 @@ def execute(self, tasks: list[str | Address | Task] | TaskGraph | None = None) -
 
         if not graph.is_complete():
             raise BuildError(list(graph.tasks(failed=True)))
+        return graph
 
     @overload
     def listen(

kraken-build/src/kraken/std/cargo/__init__.py

@@ -427,6 +427,7 @@ def cargo_publish(
     project: Project | None = None,
     version: str | None = None,
     cargo_toml_file: Path = Path("Cargo.toml"),
+    allow_overwrite: bool = False,
 ) -> CargoPublishTask:
     """Creates a task that publishes the create to the specified *registry*.
 
@@ -459,6 +460,7 @@ def cargo_publish(
     task.version = version
     task.cargo_toml_file = cargo_toml_file
     task.depends_on(f":{CARGO_PUBLISH_SUPPORT_GROUP_NAME}?")
+    task.allow_overwrite = allow_overwrite
     return task
 
 

kraken-build/src/kraken/std/cargo/tasks/cargo_publish_task.py

@@ -1,11 +1,16 @@
 import contextlib
+import json
 import logging
 from pathlib import Path
 from typing import Any
 
+import requests
+import requests.auth
+
 from kraken.common import atomic_file_swap, not_none
 from kraken.core import Project, Property, TaskStatus
 from kraken.std.cargo import CargoProject
+from kraken.std.cargo.manifest import CargoManifest
 
 from ..config import CargoRegistry
 from .cargo_build_task import CargoBuildTask
@@ -37,6 +42,38 @@ class CargoPublishTask(CargoBuildTask):
     #: Cargo.toml which to temporarily bump
     cargo_toml_file: Property[Path] = Property.default("Config.toml")
 
+    #: Allow Overwrite of existing packages
+    allow_overwrite: Property[bool] = Property.default(False)
+
+    def prepare(self) -> TaskStatus | None:
+        """Checks if the crate@version already exists in the registry. If so, the task will be skipped"""
+        if self.allow_overwrite.get():
+            return TaskStatus.pending()
+
+        manifest = CargoManifest.read(self.cargo_toml_file.get())
+        manifest_package = manifest.package
+        manifest_package_name = manifest_package.name if manifest_package is not None else None
+        manifest_version = manifest_package.version if manifest_package is not None else None
+
+        package_name = self.package_name.get() or manifest_package_name
+        version = self.version.get() or manifest_version
+
+        if not package_name:
+            return TaskStatus.pending("Unable to verify package existence - unknown package name")
+        if not version:
+            return TaskStatus.pending("Unable to verify package existence - unknown version")
+
+        try:
+            return self._check_package_existence(package_name, version, self.registry.get())
+        except Exception as e:
+            logger.warn(
+                "An error happened while checking for {} existence in %s, %s",
+                package_name,
+                self.registry.get().alias,
+                e,
+            )
+            return TaskStatus.pending("Unable to verify package existence")
+
     def get_cargo_command(self, env: dict[str, str]) -> list[str]:
         super().get_cargo_command(env)
         registry = self.registry.get()
@@ -65,15 +102,11 @@ def __init__(self, name: str, project: Project) -> None:
         self._base_command = ["cargo", "publish"]
 
     def _get_updated_cargo_toml(self, version: str) -> str:
-        from kraken.std.cargo.manifest import CargoManifest
-
         manifest = CargoManifest.read(self.cargo_toml_file.get())
         if manifest.package is None:
             return manifest.to_toml_string()
 
-        # Cargo does not play nicely with semver metadata (ie. 1.0.1-dev3+abc123)
-        # We replace that to 1.0.1-dev3abc123
-        fixed_version_string = version.replace("+", "")
+        fixed_version_string = self._sanitize_version(version)
         manifest.package.version = fixed_version_string
         if manifest.workspace and manifest.workspace.package:
             manifest.workspace.package.version = version
@@ -108,3 +141,79 @@ def execute(self) -> TaskStatus:
                 fp.close()
             result = super().execute()
         return result
+
+    @staticmethod
+    def _sanitize_version(version: str) -> str:
+        """
+        Cargo does not play nicely with semver metadata (ie. 1.0.1-dev3+abc123)
+        We replace that to 1.0.1-dev3abc123
+        """
+        return version.replace("+", "")
+
+    @classmethod
+    def _check_package_existence(cls, package_name: str, version: str, registry: CargoRegistry) -> TaskStatus | None:
+        """
+        Checks wether the given `package_name`@`version` is indexed in the provided `registry`.
+
+        Checking is done by reading from the registry's index HTTP API, following the
+        [Index Format](https://doc.rust-lang.org/cargo/reference/registry-index.html) documentation
+        """
+        if not registry.index.startswith("sparse+"):
+            return TaskStatus.pending("Unable to verify package existence - Only sparse registries are supported")
+        index = registry.index.removeprefix("sparse+")
+        index = index.removesuffix("/")
+
+        # >> Index authentication
+        session = requests.sessions.Session()
+        config_response = session.get(f"{index}/config.json")
+        if config_response.status_code == 401:
+            if registry.read_credentials is None:
+                return TaskStatus.pending(
+                    "Unable to verify package existence - registry requires authentication, but no credentials set"
+                )
+            session.auth = registry.read_credentials
+            config_response = session.get(f"{index}/config.json")
+            if config_response.status_code % 200 != 0:
+                logger.warn(config_response.text)
+                return TaskStatus.pending(
+                    "Unable to verify package existence - failed to download config.json file from registry"
+                )
+
+        # >> Index files layout
+        # Reference: https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files
+        path = []
+        if len(package_name) == 1:
+            path = ["1"]
+        elif len(package_name) == 2:
+            path = ["2"]
+        elif len(package_name) == 3:
+            path = ["3", package_name.lower()[0]]
+        else:
+            package_name_lower = package_name.lower()
+            path = [package_name_lower[0:2], package_name_lower[2:4]]
+
+        # >> Download the index file
+        index_path = "/".join(path + [package_name])
+        index_response = session.get(f"{index}/{index_path}")
+
+        if index_response.status_code in [404, 410, 451]:
+            return TaskStatus.pending(f"Package {package_name} does not already exists in {registry.alias}")
+        elif index_response.status_code % 200 != 0:
+            logger.warn(index_response.text)
+            return TaskStatus.pending("Unable to verify package existence - error when fetching package information")
+
+        sanitized_version = cls._sanitize_version(version)
+
+        # >> Search for relevant version in the index file
+        for registry_version in index_response.text.split("\n"):
+            # Index File is sometimes newline terminated
+            if not registry_version:
+                continue
+            registry_version = cls._sanitize_version(json.loads(registry_version).get("vers", ""))
+            if registry_version == sanitized_version:
+                return TaskStatus.skipped(
+                    f"Package {package_name} with version {version} already exists in {registry.alias}"
+                )
+        return TaskStatus.pending(
+            f"Package {package_name} with version {version} does not already exists in {registry.alias}"
+        )

kraken-build/tests/kraken_std/integration/cargo/test_cargo_private_registry.py

@@ -13,6 +13,7 @@
 from pathlib import Path
 
 import pytest
+from requests_mock import Mocker
 
 from kraken.core import BuildError
 from kraken.core.testing import kraken_ctx, kraken_project
@@ -38,6 +39,41 @@ class CargoRepositoryWithAuth:
     token: str | None
 
 
+def skip_publish_lib(repository: CargoRepositoryWithAuth, tempdir: Path) -> None:
+    lib_dir = tempdir.joinpath("cargo-hello-world-lib")
+    shutil.copytree(example_dir("cargo-hello-world-lib"), lib_dir)
+    cargo_registry_id = "private-repo"
+
+    with unittest.mock.patch.dict(os.environ, {"CARGO_HOME": str(tempdir)}):
+        # Build the library and publish it to the registry.
+        logger.info(
+            "Publishing cargo-hello-world-lib to Cargo repository %r (%r)",
+            repository.name,
+            repository.index_url,
+        )
+
+        with kraken_ctx() as ctx, kraken_project(ctx) as project1:
+            project1.directory = lib_dir
+            cargo_registry(
+                cargo_registry_id,
+                repository.index_url,
+                read_credentials=repository.creds,
+                publish_token=repository.token,
+            )
+            cargo_auth_proxy()
+            task = cargo_sync_config()
+            task.git_fetch_with_cli.set(True)
+            cargo_check_toolchain_version(minimal_version="1.60")
+            publish_task = cargo_publish(
+                cargo_registry_id,
+                version="0.1.0",
+                cargo_toml_file=project1.directory.joinpath("Cargo.toml"),
+            )
+            graph = project1.context.execute(["publish"])
+            status = graph.get_status(publish_task)
+            assert status is not None and status.is_skipped()
+
+
 def publish_lib_and_build_app(repository: CargoRepositoryWithAuth, tempdir: Path) -> None:
     # Copy the Cargo project files to a temporary directory.
     for item in ["cargo-hello-world-lib", "cargo-hello-world-app"]:
@@ -157,3 +193,14 @@ def test__private_cargo_registry_publish_and_consume(tempdir: Path, private_regi
         "kraken-std-cargo-integration-test", private_registry, None, "xxxxxxxxxxxxxxxxxxxxxx"
     )
     publish_lib_and_build_app(repository, tempdir)
+
+
+def test__mock_cargo_registry_skips_publish_if_exists(tempdir: Path, requests_mock: Mocker) -> None:
+    registry_url = "http://0.0.0.0:35510"
+    index_url = f"sparse+{registry_url}/"
+
+    requests_mock.get(f"{registry_url}/config.json", text="{}")
+    requests_mock.get(f"{registry_url}/he/ll/hello-world-lib", text='{"vers": "0.1.0"}')
+
+    repository = CargoRepositoryWithAuth("kraken-std-cargo-integration-test", index_url, None, "xxxxxxxxxxxxxxxxxxxxxx")
+    skip_publish_lib(repository, tempdir)