diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 199feb983..f8f67056c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: fix-byte-order-marker - id: mixed-line-ending - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.9.1" + rev: "v0.9.2" hooks: # Run the linter - id: ruff diff --git a/charmcraft/application/commands/store.py b/charmcraft/application/commands/store.py index 04700b6b5..0d0918ffb 100644 --- a/charmcraft/application/commands/store.py +++ b/charmcraft/application/commands/store.py @@ -1630,18 +1630,20 @@ def run(self, parsed_args: argparse.Namespace) -> None: # get tips from the Store store = Store(env.get_store_config(), needs_auth=False) - to_query = [] - for lib in local_libs_data: - if lib.lib_id is None: - item = { - "charm_name": lib.charm_name, - "lib_name": lib.lib_name, - "api": lib.api, - } - else: - item = {"lib_id": lib.lib_id, "api": lib.api} - to_query.append(item) - libs_tips = store.get_libraries_tips(to_query) + try: + libs_tips = self._services.store.get_libraries_metadata( + [ + project.CharmLib( + lib=f"{lib.charm_name}.{lib.lib_name}", version=str(lib.api) + ) + for lib in local_libs_data + ] + ) + except errors.LibraryError: + raise errors.LibraryError( + message=f"Library {parsed_args.library} not found in Charmhub.", + logpath_report=False, + ) # check if something needs to be done analysis = [] @@ -1649,7 +1651,7 @@ def run(self, parsed_args: argparse.Namespace) -> None: emit.debug(f"Verifying local lib {lib_data}") # fix any missing lib id using the Store info if lib_data.lib_id is None: - for tip in libs_tips.values(): + for tip in libs_tips: if ( lib_data.charm_name == tip.charm_name and lib_data.lib_name == tip.lib_name @@ -1657,18 +1659,20 @@ def run(self, parsed_args: argparse.Namespace) -> None: lib_data = dataclasses.replace(lib_data, lib_id=tip.lib_id) break - tip = libs_tips.get((lib_data.lib_id, lib_data.api)) + for tip in libs_tips: + if tip.lib_id == lib_data.lib_id and tip.api == lib_data.api: + break + else: + raise errors.LibraryError( + message=f"Library {parsed_args.library} not found in Charmhub.", + logpath_report=False, + ) emit.debug(f"Store tip: {tip}") error_message = None - if tip is None: - error_message = f"Library {lib_data.full_name} not found in Charmhub." - elif tip.patch > lib_data.patch: - # the store has a higher version than local - pass - elif tip.patch < lib_data.patch: + if tip.patch < lib_data.patch: # the store has a lower version numbers than local error_message = f"Library {lib_data.full_name} has local changes, cannot be updated." - else: + elif tip.patch == lib_data.patch: # same versions locally and in the store if tip.content_hash == lib_data.content_hash: error_message = ( diff --git a/requirements-dev.txt b/requirements-dev.txt index d135a0fdd..e50fadaf4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,7 +9,7 @@ cffi==1.17.1 chardet==5.2.0 charset-normalizer==3.4.1 coverage==7.6.10 -craft-application==4.8.1 +craft-application==4.8.2 craft-archives==2.0.2 craft-cli==2.13.0 craft-grammar==2.0.1 @@ -59,7 +59,7 @@ protobuf==5.29.3 pycparser==2.22 pydantic==2.9.2 pydantic-core==2.23.4 -pyfakefs==5.7.3 +pyfakefs==5.7.4 pygit2==1.14.1 pylint==3.3.3 pymacaroons==0.13.0 diff --git a/tests/integration/commands/test_store_commands.py b/tests/integration/commands/test_store_commands.py index a49f3c352..1cb23944c 100644 --- a/tests/integration/commands/test_store_commands.py +++ b/tests/integration/commands/test_store_commands.py @@ -17,18 +17,25 @@ import argparse import datetime +import json +import pathlib +import re import sys from unittest import mock +import craft_cli.pytest_plugin import pytest from craft_store import publisher -from charmcraft import env +from charmcraft import env, errors, utils from charmcraft.application.commands import FetchLibCommand from charmcraft.application.commands.store import CreateTrack -from charmcraft.store.models import Library from tests import factory +OPERATOR_LIBS_LINUX_APT_ID = "7c3dbc9c2ad44a47bd6fcb25caa270e5" +OPERATOR_LIBS_LINUX_SNAP_ID = "05394e5893f94f2d90feb7cbe6b633cd" +MYSQL_MYSQL_ID = "8c1428f06b1b4ec8bf98b7d980a38a8c" + @pytest.fixture def store_mock(): @@ -47,469 +54,235 @@ def validate_params(config, ephemeral=False, needs_auth=True): # region fetch-lib tests +@pytest.mark.slow @pytest.mark.parametrize("formatted", [None, "json"]) def test_fetchlib_simple_downloaded( - emitter, store_mock, tmp_path, monkeypatch, config, formatted + emitter: craft_cli.pytest_plugin.RecordingEmitter, + new_path: pathlib.Path, + config, + formatted: str | None, ): """Happy path fetching the lib for the first time (downloading it).""" - monkeypatch.chdir(tmp_path) - - lib_id = "test-example-lib-id" - lib_content = "some test content with uñicode ;)" - store_mock.get_libraries_tips.return_value = { - (lib_id, 0): Library( - lib_id=lib_id, - content=None, - content_hash="abc", - api=0, - patch=7, - lib_name="testlib", - charm_name="testcharm", - ), - } - store_mock.get_library.return_value = Library( - lib_id=lib_id, - content=lib_content, - content_hash="abc", - api=0, - patch=7, - lib_name="testlib", - charm_name="testcharm", + saved_file = new_path / utils.get_lib_path("operator_libs_linux", "apt", 0) + args = argparse.Namespace( + library="charms.operator_libs_linux.v0.apt", format=formatted ) - - args = argparse.Namespace(library="charms.testcharm.v0.testlib", format=formatted) FetchLibCommand(config).run(args) - assert store_mock.mock_calls == [ - mock.call.get_libraries_tips( - [{"charm_name": "testcharm", "lib_name": "testlib", "api": 0}] - ), - mock.call.get_library("testcharm", lib_id, 0), - ] - if formatted: - expected = [ - { - "charm_name": "testcharm", - "library_name": "testlib", - "library_id": lib_id, - "api": 0, - "fetched": { - "patch": 7, - "content_hash": "abc", - }, - }, - ] - emitter.assert_json_output(expected) - else: - expected = "Library charms.testcharm.v0.testlib version 0.7 downloaded." - emitter.assert_message(expected) - saved_file = tmp_path / "lib" / "charms" / "testcharm" / "v0" / "testlib.py" - assert saved_file.read_text() == lib_content + assert saved_file.exists() + message = emitter.interactions[-1].args[1] -def test_fetchlib_simple_dash_in_name( - emitter, store_mock, tmp_path, monkeypatch, config -): - """Happy path fetching the lib for the first time (downloading it).""" - monkeypatch.chdir(tmp_path) - - lib_id = "test-example-lib-id" - lib_content = "some test content with uñicode ;)" - store_mock.get_libraries_tips.return_value = { - (lib_id, 0): Library( - lib_id=lib_id, - content=None, - content_hash="abc", - api=0, - patch=7, - lib_name="testlib", - charm_name="test-charm", - ), - } - store_mock.get_library.return_value = Library( - lib_id=lib_id, - content=lib_content, - content_hash="abc", - api=0, - patch=7, - lib_name="testlib", - charm_name="test-charm", - ) - - args = argparse.Namespace(library="charms.test_charm.v0.testlib", format=None) - FetchLibCommand(config).run(args) + if formatted: + message_dict = json.loads(message)[0] + assert isinstance(message_dict["fetched"]["patch"], int) + assert len(message_dict["fetched"]["content_hash"]) == 64 # sha256 hash + del message_dict["fetched"] + assert message_dict == { + "charm_name": "operator-libs-linux", + "library_name": "apt", + "library_id": OPERATOR_LIBS_LINUX_APT_ID, + "api": 0, + } + else: + assert re.match( + r"Library charms\.operator_libs_linux\.v0\.apt version 0.[0-9]+ downloaded.", + message, + ) - assert store_mock.mock_calls == [ - mock.call.get_libraries_tips( - [{"charm_name": "test-charm", "lib_name": "testlib", "api": 0}] - ), - mock.call.get_library("test-charm", lib_id, 0), - ] - expected = "Library charms.test_charm.v0.testlib version 0.7 downloaded." - emitter.assert_message(expected) - saved_file = tmp_path / "lib" / "charms" / "test_charm" / "v0" / "testlib.py" - assert saved_file.read_text() == lib_content + lib = utils.get_lib_info(lib_path=saved_file) + assert lib.api == 0 + assert lib.charm_name == "operator-libs-linux" + assert lib.lib_name == "apt" + assert lib.lib_id == OPERATOR_LIBS_LINUX_APT_ID + assert lib.patch > 1 -def test_fetchlib_simple_dash_in_name_on_disk( - emitter, store_mock, tmp_path, monkeypatch, config +@pytest.mark.slow +def test_fetchlib_simple_updated( + emitter: craft_cli.pytest_plugin.RecordingEmitter, new_path: pathlib.Path, config ): - """Happy path fetching the lib for the first time (downloading it).""" - monkeypatch.chdir(tmp_path) - - lib_id = "test-example-lib-id" - lib_content = "test-content" - store_mock.get_libraries_tips.return_value = { - (lib_id, 0): Library( - lib_id=lib_id, - content=None, - content_hash="abc", - api=0, - patch=7, - lib_name="testlib", - charm_name="test-charm", - ), - } - store_mock.get_library.return_value = Library( - lib_id=lib_id, - content=lib_content, - content_hash="abc", - api=0, - patch=7, - lib_name="testlib", - charm_name="test-charm", + """Happy path fetching the lib for Nth time (updating it).""" + content, content_hash = factory.create_lib_filepath( + "operator-libs-linux", "apt", api=0, patch=1, lib_id=OPERATOR_LIBS_LINUX_APT_ID ) - factory.create_lib_filepath("test-charm", "testlib", api=0, patch=1, lib_id=lib_id) - args = argparse.Namespace(library=None, format=None) + args = argparse.Namespace(library="charms.operator_libs_linux.v0.apt", format=None) FetchLibCommand(config).run(args) - assert store_mock.mock_calls == [ - mock.call.get_libraries_tips([{"lib_id": "test-example-lib-id", "api": 0}]), - mock.call.get_library("test-charm", lib_id, 0), - ] - expected = "Library charms.test_charm.v0.testlib updated to version 0.7." - emitter.assert_message(expected) - - -def test_fetchlib_simple_updated(emitter, store_mock, tmp_path, monkeypatch, config): - """Happy path fetching the lib for Nth time (updating it).""" - monkeypatch.chdir(tmp_path) + message = emitter.interactions[-1].args[1] - lib_id = "test-example-lib-id" - content, content_hash = factory.create_lib_filepath( - "testcharm", "testlib", api=0, patch=1, lib_id=lib_id + assert re.match( + r"Library charms\.operator_libs_linux\.v0\.apt updated to version 0\.[0-9]+\.", + message, ) - new_lib_content = "some test content with uñicode ;)" - store_mock.get_libraries_tips.return_value = { - (lib_id, 0): Library( - lib_id=lib_id, - content=None, - content_hash="abc", - api=0, - patch=2, - lib_name="testlib", - charm_name="testcharm", - ), - } - store_mock.get_library.return_value = Library( - lib_id=lib_id, - content=new_lib_content, - content_hash="abc", - api=0, - patch=2, - lib_name="testlib", - charm_name="testcharm", - ) - - args = argparse.Namespace(library="charms.testcharm.v0.testlib", format=None) - FetchLibCommand(config).run(args) - - assert store_mock.mock_calls == [ - mock.call.get_libraries_tips([{"lib_id": lib_id, "api": 0}]), - mock.call.get_library("testcharm", lib_id, 0), - ] - expected = "Library charms.testcharm.v0.testlib updated to version 0.2." - emitter.assert_message(expected) - saved_file = tmp_path / "lib" / "charms" / "testcharm" / "v0" / "testlib.py" - assert saved_file.read_text() == new_lib_content + saved_file = new_path / utils.get_lib_path("operator_libs_linux", "apt", 0) + lib = utils.get_lib_info(lib_path=saved_file) + assert lib.api == 0 + assert lib.charm_name == "operator-libs-linux" + assert lib.lib_name == "apt" + assert lib.lib_id == OPERATOR_LIBS_LINUX_APT_ID + assert lib.patch > 1 +@pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows not [yet] supported") @pytest.mark.parametrize("formatted", [None, "json"]) -def test_fetchlib_all(emitter, store_mock, tmp_path, monkeypatch, config, formatted): +def test_fetchlib_all( + emitter: craft_cli.pytest_plugin.RecordingEmitter, + new_path: pathlib.Path, + config, + formatted: str | None, +): """Update all the libraries found in disk.""" - monkeypatch.chdir(tmp_path) - - c1, h1 = factory.create_lib_filepath( - "testcharm1", "testlib1", api=0, patch=1, lib_id="lib_id_1" - ) - c2, h2 = factory.create_lib_filepath( - "testcharm2", "testlib2", api=3, patch=5, lib_id="lib_id_2" + factory.create_lib_filepath( + "operator-libs-linux", + "snap", + api=0, + patch=1, + lib_id=OPERATOR_LIBS_LINUX_SNAP_ID, ) - - store_mock.get_libraries_tips.return_value = { - ("lib_id_1", 0): Library( - lib_id="lib_id_1", - content=None, - content_hash="abc", - api=0, - patch=2, - lib_name="testlib1", - charm_name="testcharm1", - ), - ("lib_id_2", 3): Library( - lib_id="lib_id_2", - content=None, - content_hash="def", - api=3, - patch=14, - lib_name="testlib2", - charm_name="testcharm2", - ), - } - _store_libs_info = [ - Library( - lib_id="lib_id_1", - content="new lib content 1", - content_hash="xxx", - api=0, - patch=2, - lib_name="testlib1", - charm_name="testcharm1", - ), - Library( - lib_id="lib_id_2", - content="new lib content 2", - content_hash="yyy", - api=3, - patch=14, - lib_name="testlib2", - charm_name="testcharm2", - ), - ] - store_mock.get_library.side_effect = lambda *a: _store_libs_info.pop(0) + factory.create_lib_filepath("mysql", "mysql", api=0, patch=1, lib_id=MYSQL_MYSQL_ID) args = argparse.Namespace(library=None, format=formatted) FetchLibCommand(config).run(args) + message = emitter.interactions[-1].args[1] - assert store_mock.mock_calls == [ - mock.call.get_libraries_tips( - [ - {"lib_id": "lib_id_1", "api": 0}, - {"lib_id": "lib_id_2", "api": 3}, - ] - ), - mock.call.get_library("testcharm1", "lib_id_1", 0), - mock.call.get_library("testcharm2", "lib_id_2", 3), - ] - names = [ - "charms.testcharm1.v0.testlib1", - "charms.testcharm2.v3.testlib2", - ] - emitter.assert_debug("Libraries found under 'lib/charms': " + str(names)) if formatted: - expected = [ + message_list = json.loads(message) + for message_dict in message_list: + assert isinstance(message_dict["fetched"]["patch"], int) + assert len(message_dict["fetched"]["content_hash"]) == 64 # sha256 hash + del message_dict["fetched"] + assert message_list == [ { - "charm_name": "testcharm1", - "library_name": "testlib1", - "library_id": "lib_id_1", + "charm_name": "mysql", + "library_name": "mysql", + "library_id": MYSQL_MYSQL_ID, "api": 0, - "fetched": { - "patch": 2, - "content_hash": "xxx", - }, }, { - "charm_name": "testcharm2", - "library_name": "testlib2", - "library_id": "lib_id_2", - "api": 3, - "fetched": { - "patch": 14, - "content_hash": "yyy", - }, + "charm_name": "operator-libs-linux", + "library_name": "snap", + "library_id": OPERATOR_LIBS_LINUX_SNAP_ID, + "api": 0, }, ] - emitter.assert_json_output(expected) else: - emitter.assert_messages( - [ - "Library charms.testcharm1.v0.testlib1 updated to version 0.2.", - "Library charms.testcharm2.v3.testlib2 updated to version 3.14.", - ] + assert re.match( + r"Library charms\.[a-z_]+\.v0\.[a-z]+ updated to version 0\.[0-9]+\.", + message, ) - saved_file = tmp_path / "lib" / "charms" / "testcharm1" / "v0" / "testlib1.py" - assert saved_file.read_text() == "new lib content 1" - saved_file = tmp_path / "lib" / "charms" / "testcharm2" / "v3" / "testlib2.py" - assert saved_file.read_text() == "new lib content 2" + saved_file = new_path / utils.get_lib_path("operator_libs_linux", "snap", 0) + lib = utils.get_lib_info(lib_path=saved_file) + assert lib.api == 0 + assert lib.charm_name == "operator-libs-linux" + assert lib.lib_name == "snap" + assert lib.lib_id == OPERATOR_LIBS_LINUX_SNAP_ID + assert lib.patch > 1 + + saved_file = new_path / utils.get_lib_path("mysql", "mysql", 0) + lib = utils.get_lib_info(lib_path=saved_file) + assert lib.api == 0 + assert lib.charm_name == "mysql" + assert lib.lib_name == "mysql" + assert lib.lib_id == MYSQL_MYSQL_ID + assert lib.patch > 1 +@pytest.mark.slow @pytest.mark.parametrize("formatted", [None, "json"]) -def test_fetchlib_store_not_found(emitter, store_mock, config, formatted): +def test_fetchlib_store_not_found( + emitter: craft_cli.pytest_plugin.RecordingEmitter, + new_path: pathlib.Path, + config, + formatted: str | None, +) -> None: """The indicated library is not found in the store.""" - store_mock.get_libraries_tips.return_value = {} args = argparse.Namespace(library="charms.testcharm.v0.testlib", format=formatted) - FetchLibCommand(config).run(args) - ( - store_mock.get_libraries_tips.assert_called_once_with( - [{"charm_name": "testcharm", "lib_name": "testlib", "api": 0}] - ), + with pytest.raises(errors.LibraryError) as exc_info: + FetchLibCommand(config).run(args) + + assert exc_info.value.args[0] == ( + "Library charms.testcharm.v0.testlib not found in Charmhub." ) - error_message = "Library charms.testcharm.v0.testlib not found in Charmhub." - if formatted: - expected = [ - { - "charm_name": "testcharm", - "library_name": "testlib", - "library_id": None, - "api": 0, - "error_message": error_message, - }, - ] - emitter.assert_json_output(expected) - else: - emitter.assert_message(error_message) +@pytest.mark.slow @pytest.mark.parametrize("formatted", [None, "json"]) def test_fetchlib_store_is_old( - emitter, store_mock, tmp_path, monkeypatch, config, formatted + emitter: craft_cli.pytest_plugin.RecordingEmitter, + new_path: pathlib.Path, + config, + formatted: str | None, ): """The store has an older version that what is found locally.""" - monkeypatch.chdir(tmp_path) - - lib_id = "test-example-lib-id" - factory.create_lib_filepath("testcharm", "testlib", api=0, patch=7, lib_id=lib_id) - - store_mock.get_libraries_tips.return_value = { - (lib_id, 0): Library( - lib_id=lib_id, - content=None, - content_hash="abc", - api=0, - patch=6, - lib_name="testlib", - charm_name="testcharm", - ), - } - args = argparse.Namespace(library="charms.testcharm.v0.testlib", format=formatted) + factory.create_lib_filepath( + "mysql", "mysql", api=0, patch=2**63, lib_id=MYSQL_MYSQL_ID + ) + + args = argparse.Namespace(library="charms.mysql.v0.mysql", format=formatted) FetchLibCommand(config).run(args) - store_mock.get_libraries_tips.assert_called_once_with( - [{"lib_id": lib_id, "api": 0}] - ) error_message = ( - "Library charms.testcharm.v0.testlib has local changes, cannot be updated." + "Library charms.mysql.v0.mysql has local changes, cannot be updated." ) if formatted: expected = [ { - "charm_name": "testcharm", - "library_name": "testlib", - "library_id": lib_id, + "charm_name": "mysql", + "library_name": "mysql", + "library_id": MYSQL_MYSQL_ID, "api": 0, "error_message": error_message, }, ] - emitter.assert_json_output(expected) + emitter.assert_json_output( # pyright: ignore[reportAttributeAccessIssue] + expected + ) else: emitter.assert_message(error_message) -@pytest.mark.parametrize("formatted", [None, "json"]) +@pytest.mark.slow def test_fetchlib_store_same_versions_same_hash( - emitter, store_mock, tmp_path, monkeypatch, config, formatted + emitter: craft_cli.pytest_plugin.RecordingEmitter, new_path: pathlib.Path, config ): """The store situation is the same than locally.""" - monkeypatch.chdir(tmp_path) - - lib_id = "test-example-lib-id" - _, c_hash = factory.create_lib_filepath( - "testcharm", "testlib", api=0, patch=7, lib_id=lib_id - ) + args = argparse.Namespace(library="charms.operator_libs_linux.v0.snap", format=None) + # This run is a setup run + FetchLibCommand(config).run(args) - store_mock.get_libraries_tips.return_value = { - (lib_id, 0): Library( - lib_id=lib_id, - content=None, - content_hash=c_hash, - api=0, - patch=7, - lib_name="testlib", - charm_name="testcharm", - ), - } - args = argparse.Namespace(library="charms.testcharm.v0.testlib", format=formatted) + # The real run FetchLibCommand(config).run(args) - store_mock.get_libraries_tips.assert_called_once_with( - [{"lib_id": lib_id, "api": 0}] + assert re.match( + r"Library charms.operator_libs_linux.v0.snap was already up to date in version 0.[0-9]+.", + emitter.interactions[-1].args[1], ) - error_message = ( - "Library charms.testcharm.v0.testlib was already up to date in version 0.7." - ) - if formatted: - expected = [ - { - "charm_name": "testcharm", - "library_name": "testlib", - "library_id": lib_id, - "api": 0, - "error_message": error_message, - }, - ] - emitter.assert_json_output(expected) - else: - emitter.assert_message(error_message) -@pytest.mark.parametrize("formatted", [None, "json"]) +@pytest.mark.slow def test_fetchlib_store_same_versions_different_hash( - emitter, store_mock, tmp_path, monkeypatch, config, formatted + emitter: craft_cli.pytest_plugin.RecordingEmitter, new_path: pathlib.Path, config ): """The store has the lib in the same version, but with different content.""" - monkeypatch.chdir(tmp_path) - - lib_id = "test-example-lib-id" - factory.create_lib_filepath("testcharm", "testlib", api=0, patch=7, lib_id=lib_id) - - store_mock.get_libraries_tips.return_value = { - (lib_id, 0): Library( - lib_id=lib_id, - content=None, - content_hash="abc", - api=0, - patch=7, - lib_name="testlib", - charm_name="testcharm", - ), - } - args = argparse.Namespace(library="charms.testcharm.v0.testlib", format=formatted) + args = argparse.Namespace(library="charms.operator_libs_linux.v0.snap", format=None) + lib_path = utils.get_lib_path("operator-libs-linux", "snap", 0) + # This run is a setup run FetchLibCommand(config).run(args) + with lib_path.open("a+") as f: + f.write("# This changes the hash!") - assert store_mock.mock_calls == [ - mock.call.get_libraries_tips([{"lib_id": lib_id, "api": 0}]), - ] - error_message = ( - "Library charms.testcharm.v0.testlib has local changes, cannot be updated." + # The real run + FetchLibCommand(config).run(args) + + assert emitter.interactions[-1].args[1] == ( + "Library charms.operator_libs_linux.v0.snap has local changes, cannot be updated." ) - if formatted: - expected = [ - { - "charm_name": "testcharm", - "library_name": "testlib", - "library_id": lib_id, - "api": 0, - "error_message": error_message, - }, - ] - emitter.assert_json_output(expected) - else: - emitter.assert_message(error_message) # endregion