diff --git a/kraken-build/.changelog/_unreleased.toml b/kraken-build/.changelog/_unreleased.toml new file mode 100644 index 00000000..7ced0ff2 --- /dev/null +++ b/kraken-build/.changelog/_unreleased.toml @@ -0,0 +1,5 @@ +[[entries]] +id = "48a06299-f287-49b8-8c91-1f74cfa56f2f" +type = "feature" +description = "Cargo Publishing : Skip publishing packages that already exists in the registry" +author = "@jcaille" diff --git a/kraken-build/pyproject.toml b/kraken-build/pyproject.toml index debf3ab7..96224e46 100644 --- a/kraken-build/pyproject.toml +++ b/kraken-build/pyproject.toml @@ -64,6 +64,7 @@ types-requests = "^2.28.0" pytest-xdist = {version = "^3.5.0", extras = ["psutil"]} mitmproxy = "^10.2.4" types-networkx = "^3.2.1.20240703" +requests-mock = "^1.12.1" # Slap configuration # ------------------ diff --git a/kraken-build/src/kraken/core/system/context.py b/kraken-build/src/kraken/core/system/context.py index 0e8e42a3..2da2c332 100644 --- a/kraken-build/src/kraken/core/system/context.py +++ b/kraken-build/src/kraken/core/system/context.py @@ -357,7 +357,7 @@ def get_build_graph(self, targets: Sequence[str | Address | Task] | None) -> Tas assert graph, "TaskGraph cannot be empty" return graph - def execute(self, tasks: list[str | Address | Task] | TaskGraph | None = None) -> None: + def execute(self, tasks: list[str | Address | Task] | TaskGraph | None = None) -> TaskGraph: """Execute all default tasks or the tasks specified by *targets* using the default executor. If :meth:`finalize` was not called already it will be called by this function before the build graph is created, unless a build graph is passed in the first place. @@ -379,6 +379,7 @@ def execute(self, tasks: list[str | Address | Task] | TaskGraph | None = None) - if not graph.is_complete(): raise BuildError(list(graph.tasks(failed=True))) + return graph @overload def listen( diff --git a/kraken-build/src/kraken/std/cargo/__init__.py b/kraken-build/src/kraken/std/cargo/__init__.py index 4d2695ee..cd6b9b76 100644 --- a/kraken-build/src/kraken/std/cargo/__init__.py +++ b/kraken-build/src/kraken/std/cargo/__init__.py @@ -427,6 +427,7 @@ def cargo_publish( project: Project | None = None, version: str | None = None, cargo_toml_file: Path = Path("Cargo.toml"), + allow_overwrite: bool = False, ) -> CargoPublishTask: """Creates a task that publishes the create to the specified *registry*. @@ -459,6 +460,7 @@ def cargo_publish( task.version = version task.cargo_toml_file = cargo_toml_file task.depends_on(f":{CARGO_PUBLISH_SUPPORT_GROUP_NAME}?") + task.allow_overwrite = allow_overwrite return task diff --git a/kraken-build/src/kraken/std/cargo/tasks/cargo_publish_task.py b/kraken-build/src/kraken/std/cargo/tasks/cargo_publish_task.py index 3a8f8162..a57633e1 100644 --- a/kraken-build/src/kraken/std/cargo/tasks/cargo_publish_task.py +++ b/kraken-build/src/kraken/std/cargo/tasks/cargo_publish_task.py @@ -1,11 +1,16 @@ import contextlib +import json import logging from pathlib import Path from typing import Any +import requests +import requests.auth + from kraken.common import atomic_file_swap, not_none from kraken.core import Project, Property, TaskStatus from kraken.std.cargo import CargoProject +from kraken.std.cargo.manifest import CargoManifest from ..config import CargoRegistry from .cargo_build_task import CargoBuildTask @@ -37,6 +42,38 @@ class CargoPublishTask(CargoBuildTask): #: Cargo.toml which to temporarily bump cargo_toml_file: Property[Path] = Property.default("Config.toml") + #: Allow Overwrite of existing packages + allow_overwrite: Property[bool] = Property.default(False) + + def prepare(self) -> TaskStatus | None: + """Checks if the crate@version already exists in the registry. If so, the task will be skipped""" + if self.allow_overwrite.get(): + return TaskStatus.pending() + + manifest = CargoManifest.read(self.cargo_toml_file.get()) + manifest_package = manifest.package + manifest_package_name = manifest_package.name if manifest_package is not None else None + manifest_version = manifest_package.version if manifest_package is not None else None + + package_name = self.package_name.get() or manifest_package_name + version = self.version.get() or manifest_version + + if not package_name: + return TaskStatus.pending("Unable to verify package existence - unknown package name") + if not version: + return TaskStatus.pending("Unable to verify package existence - unknown version") + + try: + return self._check_package_existence(package_name, version, self.registry.get()) + except Exception as e: + logger.warn( + "An error happened while checking for {} existence in %s, %s", + package_name, + self.registry.get().alias, + e, + ) + return TaskStatus.pending("Unable to verify package existence") + def get_cargo_command(self, env: dict[str, str]) -> list[str]: super().get_cargo_command(env) registry = self.registry.get() @@ -65,15 +102,11 @@ def __init__(self, name: str, project: Project) -> None: self._base_command = ["cargo", "publish"] def _get_updated_cargo_toml(self, version: str) -> str: - from kraken.std.cargo.manifest import CargoManifest - manifest = CargoManifest.read(self.cargo_toml_file.get()) if manifest.package is None: return manifest.to_toml_string() - # Cargo does not play nicely with semver metadata (ie. 1.0.1-dev3+abc123) - # We replace that to 1.0.1-dev3abc123 - fixed_version_string = version.replace("+", "") + fixed_version_string = self._sanitize_version(version) manifest.package.version = fixed_version_string if manifest.workspace and manifest.workspace.package: manifest.workspace.package.version = version @@ -108,3 +141,79 @@ def execute(self) -> TaskStatus: fp.close() result = super().execute() return result + + @staticmethod + def _sanitize_version(version: str) -> str: + """ + Cargo does not play nicely with semver metadata (ie. 1.0.1-dev3+abc123) + We replace that to 1.0.1-dev3abc123 + """ + return version.replace("+", "") + + @classmethod + def _check_package_existence(cls, package_name: str, version: str, registry: CargoRegistry) -> TaskStatus | None: + """ + Checks wether the given `package_name`@`version` is indexed in the provided `registry`. + + Checking is done by reading from the registry's index HTTP API, following the + [Index Format](https://doc.rust-lang.org/cargo/reference/registry-index.html) documentation + """ + if not registry.index.startswith("sparse+"): + return TaskStatus.pending("Unable to verify package existence - Only sparse registries are supported") + index = registry.index.removeprefix("sparse+") + index = index.removesuffix("/") + + # >> Index authentication + session = requests.sessions.Session() + config_response = session.get(f"{index}/config.json") + if config_response.status_code == 401: + if registry.read_credentials is None: + return TaskStatus.pending( + "Unable to verify package existence - registry requires authentication, but no credentials set" + ) + session.auth = registry.read_credentials + config_response = session.get(f"{index}/config.json") + if config_response.status_code % 200 != 0: + logger.warn(config_response.text) + return TaskStatus.pending( + "Unable to verify package existence - failed to download config.json file from registry" + ) + + # >> Index files layout + # Reference: https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files + path = [] + if len(package_name) == 1: + path = ["1"] + elif len(package_name) == 2: + path = ["2"] + elif len(package_name) == 3: + path = ["3", package_name.lower()[0]] + else: + package_name_lower = package_name.lower() + path = [package_name_lower[0:2], package_name_lower[2:4]] + + # >> Download the index file + index_path = "/".join(path + [package_name]) + index_response = session.get(f"{index}/{index_path}") + + if index_response.status_code in [404, 410, 451]: + return TaskStatus.pending(f"Package {package_name} does not already exists in {registry.alias}") + elif index_response.status_code % 200 != 0: + logger.warn(index_response.text) + return TaskStatus.pending("Unable to verify package existence - error when fetching package information") + + sanitized_version = cls._sanitize_version(version) + + # >> Search for relevant version in the index file + for registry_version in index_response.text.split("\n"): + # Index File is sometimes newline terminated + if not registry_version: + continue + registry_version = cls._sanitize_version(json.loads(registry_version).get("vers", "")) + if registry_version == sanitized_version: + return TaskStatus.skipped( + f"Package {package_name} with version {version} already exists in {registry.alias}" + ) + return TaskStatus.pending( + f"Package {package_name} with version {version} does not already exists in {registry.alias}" + ) diff --git a/kraken-build/tests/kraken_std/integration/cargo/test_cargo_private_registry.py b/kraken-build/tests/kraken_std/integration/cargo/test_cargo_private_registry.py index 91b391da..ff7a18bf 100644 --- a/kraken-build/tests/kraken_std/integration/cargo/test_cargo_private_registry.py +++ b/kraken-build/tests/kraken_std/integration/cargo/test_cargo_private_registry.py @@ -13,6 +13,7 @@ from pathlib import Path import pytest +from requests_mock import Mocker from kraken.core import BuildError from kraken.core.testing import kraken_ctx, kraken_project @@ -38,6 +39,41 @@ class CargoRepositoryWithAuth: token: str | None +def skip_publish_lib(repository: CargoRepositoryWithAuth, tempdir: Path) -> None: + lib_dir = tempdir.joinpath("cargo-hello-world-lib") + shutil.copytree(example_dir("cargo-hello-world-lib"), lib_dir) + cargo_registry_id = "private-repo" + + with unittest.mock.patch.dict(os.environ, {"CARGO_HOME": str(tempdir)}): + # Build the library and publish it to the registry. + logger.info( + "Publishing cargo-hello-world-lib to Cargo repository %r (%r)", + repository.name, + repository.index_url, + ) + + with kraken_ctx() as ctx, kraken_project(ctx) as project1: + project1.directory = lib_dir + cargo_registry( + cargo_registry_id, + repository.index_url, + read_credentials=repository.creds, + publish_token=repository.token, + ) + cargo_auth_proxy() + task = cargo_sync_config() + task.git_fetch_with_cli.set(True) + cargo_check_toolchain_version(minimal_version="1.60") + publish_task = cargo_publish( + cargo_registry_id, + version="0.1.0", + cargo_toml_file=project1.directory.joinpath("Cargo.toml"), + ) + graph = project1.context.execute(["publish"]) + status = graph.get_status(publish_task) + assert status is not None and status.is_skipped() + + def publish_lib_and_build_app(repository: CargoRepositoryWithAuth, tempdir: Path) -> None: # Copy the Cargo project files to a temporary directory. for item in ["cargo-hello-world-lib", "cargo-hello-world-app"]: @@ -157,3 +193,14 @@ def test__private_cargo_registry_publish_and_consume(tempdir: Path, private_regi "kraken-std-cargo-integration-test", private_registry, None, "xxxxxxxxxxxxxxxxxxxxxx" ) publish_lib_and_build_app(repository, tempdir) + + +def test__mock_cargo_registry_skips_publish_if_exists(tempdir: Path, requests_mock: Mocker) -> None: + registry_url = "http://0.0.0.0:35510" + index_url = f"sparse+{registry_url}/" + + requests_mock.get(f"{registry_url}/config.json", text="{}") + requests_mock.get(f"{registry_url}/he/ll/hello-world-lib", text='{"vers": "0.1.0"}') + + repository = CargoRepositoryWithAuth("kraken-std-cargo-integration-test", index_url, None, "xxxxxxxxxxxxxxxxxxxxxx") + skip_publish_lib(repository, tempdir)