From 9a8e3c0730f7c424dc8d2d5a80fa20ecbfe79dc5 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 18 Jan 2023 17:44:42 +0100 Subject: [PATCH 01/44] Refactor to make internal fields private --- .../base_defaults_component.py | 8 +- kpops/components/base_components/kafka_app.py | 14 +-- .../base_components/kafka_connect.py | 28 ++--- .../base_components/kubernetes_app.py | 4 +- .../base_components/models/to_section.py | 2 +- .../base_components/pipeline_component.py | 4 +- .../producer/producer_app.py | 11 +- .../streams_bootstrap/streams/streams_app.py | 11 +- kpops/pipeline_generator/pipeline.py | 8 +- .../test_base_defaults_component.py | 16 +-- tests/components/test_kafka_app.py | 8 +- tests/components/test_kafka_connect.py | 108 +++++++++--------- tests/components/test_kubernetes_app.py | 32 +++--- tests/components/test_pipeline_component.py | 4 +- tests/components/test_producer_app.py | 10 +- tests/components/test_streams_app.py | 34 +++--- tests/pipeline/test_components/components.py | 4 +- .../components.py | 4 +- 18 files changed, 159 insertions(+), 151 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index bdf69d6d8..15e917069 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -7,7 +7,7 @@ from typing import TypeVar import typer -from pydantic import BaseConfig, BaseModel, Field +from pydantic import BaseConfig, BaseModel, Field, PrivateAttr from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -19,9 +19,9 @@ class BaseDefaultsComponent(BaseModel): _type: str = Field(..., alias="type") - enrich: bool = Field(default=False, exclude=True) - config: PipelineConfig = Field(default=..., exclude=True) - handlers: ComponentHandlers = Field(default=..., exclude=True) + _enrich: bool = PrivateAttr(default=False) + _config: PipelineConfig = PrivateAttr(default=...) + _handlers: ComponentHandlers = PrivateAttr(default=...) class Config(BaseConfig): arbitrary_types_allowed = True diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index c2fb849c5..c9ba77926 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -2,7 +2,7 @@ import logging -from pydantic import BaseModel, Extra +from pydantic import BaseModel, Extra, Field from typing_extensions import override from kpops.component_handlers.helm_wrapper.helm import Helm @@ -20,7 +20,7 @@ class KafkaStreamsConfig(BaseModel): brokers: str - schema_registry_url: str | None = None + schema_registry_url: str | None = Field(None, title="Schema Registry URL") class Config(CamelCaseConfig): extra = Extra.allow @@ -43,12 +43,12 @@ class KafkaApp(KubernetesApp): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self.app.streams.brokers = substitute( - self.app.streams.brokers, {"broker": self.config.broker} + self.app.streams.brokers, {"broker": self._config.broker} ) if self.app.streams.schema_registry_url: self.app.streams.schema_registry_url = substitute( self.app.streams.schema_registry_url, - {"schema_registry_url": self.config.schema_registry_url}, + {"schema_registry_url": self._config.schema_registry_url}, ) @property @@ -58,12 +58,12 @@ def clean_up_helm_chart(self) -> str: @override def deploy(self, dry_run: bool) -> None: if self.to: - self.handlers.topic_handler.create_topics( + self._handlers.topic_handler.create_topics( to_section=self.to, dry_run=dry_run ) - if self.handlers.schema_handler: - self.handlers.schema_handler.submit_schemas( + if self._handlers.schema_handler: + self._handlers.schema_handler.submit_schemas( to_section=self.to, dry_run=dry_run ) super().deploy(dry_run) diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index f126dce65..22c5bfaee 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -1,7 +1,7 @@ import json import os from abc import ABC -from typing import NoReturn +from typing import Literal, NoReturn from typing_extensions import override @@ -34,37 +34,38 @@ def prepare_connector_config(self) -> None: @override def deploy(self, dry_run: bool) -> None: if self.to: - self.handlers.topic_handler.create_topics( + self._handlers.topic_handler.create_topics( to_section=self.to, dry_run=dry_run ) - if self.handlers.schema_handler: - self.handlers.schema_handler.submit_schemas( + if self._handlers.schema_handler: + self._handlers.schema_handler.submit_schemas( to_section=self.to, dry_run=dry_run ) - self.handlers.connector_handler.create_connector( + self._handlers.connector_handler.create_connector( connector_name=self.name, kafka_connect_config=self.app, dry_run=dry_run ) @override def destroy(self, dry_run: bool) -> None: - self.handlers.connector_handler.destroy_connector( + self._handlers.connector_handler.destroy_connector( connector_name=self.name, dry_run=dry_run ) @override def clean(self, dry_run: bool) -> None: if self.to: - if self.handlers.schema_handler: - self.handlers.schema_handler.delete_schemas( + if self._handlers.schema_handler: + self._handlers.schema_handler.delete_schemas( to_section=self.to, dry_run=dry_run ) - self.handlers.topic_handler.delete_topics(self.to, dry_run=dry_run) + self._handlers.topic_handler.delete_topics(self.to, dry_run=dry_run) class KafkaSourceConnector(KafkaConnector): _type = "kafka-source-connector" + type: Literal["kafka-source-connector"] = "kafka-source-connector" @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -80,17 +81,18 @@ def clean(self, dry_run: bool) -> None: self.__run_kafka_connect_resetter(dry_run) def __run_kafka_connect_resetter(self, dry_run: bool) -> None: - self.handlers.connector_handler.clean_connector( + self._handlers.connector_handler.clean_connector( connector_name=self.name, connector_type=KafkaConnectorType.SOURCE, dry_run=dry_run, - retain_clean_jobs=self.config.retain_clean_jobs, + retain_clean_jobs=self._config.retain_clean_jobs, offset_topic=os.environ[f"{ENV_PREFIX}KAFKA_CONNECT_RESETTER_OFFSET_TOPIC"], ) class KafkaSinkConnector(KafkaConnector): _type = "kafka-sink-connector" + type: Literal["kafka-sink-connector"] = "kafka-sink-connector" @override def add_input_topics(self, topics: list[str]) -> None: @@ -119,10 +121,10 @@ def clean(self, dry_run: bool) -> None: def __clean_sink_connector( self, dry_run: bool, delete_consumer_group: bool ) -> None: - self.handlers.connector_handler.clean_connector( + self._handlers.connector_handler.clean_connector( connector_name=self.name, connector_type=KafkaConnectorType.SINK, dry_run=dry_run, - retain_clean_jobs=self.config.retain_clean_jobs, + retain_clean_jobs=self._config.retain_clean_jobs, delete_consumer_group=delete_consumer_group, ) diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 451ea6b58..5250c34e5 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -49,7 +49,7 @@ def __init__(self, **kwargs): @cached_property def helm(self) -> Helm: - helm = Helm(self.config.helm_config) + helm = Helm(self._config.helm_config) if self.helm_repo_config is not None: helm.add_repo( self.helm_repo_config.repository_name, @@ -60,7 +60,7 @@ def helm(self) -> Helm: @cached_property def helm_diff(self) -> HelmDiff: - return HelmDiff(self.config.helm_diff_config) + return HelmDiff(self._config.helm_diff_config) @property def helm_release_name(self) -> str: diff --git a/kpops/components/base_components/models/to_section.py b/kpops/components/base_components/models/to_section.py index c3be1caa5..82502a660 100644 --- a/kpops/components/base_components/models/to_section.py +++ b/kpops/components/base_components/models/to_section.py @@ -23,7 +23,7 @@ class TopicConfig(BaseModel): value_schema: str | None = Field(default=None, alias="valueSchema") partitions_count: int | None = None replication_factor: int | None = None - configs: dict[str, str] = {} + configs: dict[str, str | int] = {} role: str | None = None class Config(BaseConfig): diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 4003c6621..6ef4ae71e 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -59,12 +59,12 @@ def substitute_component_variables(self, topic_name: str) -> str: :return: final topic name """ error_topic_name = self.substitute_component_names( - self.config.topic_name_config.default_error_topic_name, + self._config.topic_name_config.default_error_topic_name, self._type, **os.environ, ) output_topic_name = self.substitute_component_names( - self.config.topic_name_config.default_output_topic_name, + self._config.topic_name_config.default_output_topic_name, self._type, **os.environ, ) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 9eaeaa427..71281685d 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import BaseConfig, Extra from typing_extensions import override @@ -19,6 +21,7 @@ class ProducerApp(KafkaApp): """ _type = "producer" + type: Literal["producer"] = "producer" app: ProducerValues class Config(BaseConfig): @@ -42,22 +45,22 @@ def add_extra_output_topic(self, topic_name: str, role: str) -> None: @override def get_helm_chart(self) -> str: - return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.PRODUCER_APP.value}" + return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.PRODUCER_APP.value}" @property @override def clean_up_helm_chart(self) -> str: - return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" + return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" @property @override def helm_repo_config(self) -> HelmRepoConfig | None: - return self.config.streams_bootstrap_helm_config + return self._config.streams_bootstrap_helm_config @override def clean(self, dry_run: bool) -> None: self._run_clean_up_job( values=self.to_helm_values(), dry_run=dry_run, - retain_clean_jobs=self.config.retain_clean_jobs, + retain_clean_jobs=self._config.retain_clean_jobs, ) diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index b46957a20..95f799fe8 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Literal + from pydantic import BaseConfig, Extra from typing_extensions import override @@ -15,6 +17,7 @@ class StreamsApp(KafkaApp): """ _type = "streams-app" + type: Literal["streams-app"] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): @@ -54,17 +57,17 @@ def add_extra_output_topic(self, topic_name: str, role: str) -> None: @override def get_helm_chart(self) -> str: - return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.STREAMS_APP.value}" + return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.STREAMS_APP.value}" @property @override def helm_repo_config(self) -> HelmRepoConfig | None: - return self.config.streams_bootstrap_helm_config + return self._config.streams_bootstrap_helm_config @property @override def clean_up_helm_chart(self) -> str: - return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" + return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" @override def reset(self, dry_run: bool) -> None: @@ -80,7 +83,7 @@ def __run_streams_clean_up_job(self, dry_run: bool, delete_output: bool) -> None self._run_clean_up_job( values=values, dry_run=dry_run, - retain_clean_jobs=self.config.retain_clean_jobs, + retain_clean_jobs=self._config.retain_clean_jobs, ) def __substitute_autoscaling_topic_names(self) -> None: diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index daa8be3f1..e324b885e 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -200,7 +200,7 @@ def enrich_component( env_components_index: dict[str, dict], ) -> PipelineComponent: component_object: PipelineComponent = component_class( - handlers=self.handlers, config=self.config, **component + _handlers=self.handlers, _config=self.config, **component ) env_component_definition = env_components_index.get(component_object.name, {}) pair = update_nested_pair( @@ -212,9 +212,9 @@ def enrich_component( component_object, pair ) return component_class( - enrich=False, - handlers=self.handlers, - config=self.config, + _enrich=False, + _handlers=self.handlers, + _config=self.config, **json_object, ) diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index 51053206b..3e9341d64 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -57,8 +57,8 @@ def test_inherit_defaults( self, config: PipelineConfig, handlers: ComponentHandlers ): component = TestChildModel( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, ) assert ( @@ -79,8 +79,8 @@ def test_inherit_defaults( def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): component = TestChildModel( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{"name": "name-defined-in-pipeline_generator"} ) @@ -104,8 +104,8 @@ def test_multiple_generations( self, config: PipelineConfig, handlers: ComponentHandlers ): component = TestGrandChildModel( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, ) assert ( @@ -134,8 +134,8 @@ class TestEnvVarModel(BaseDefaultsComponent): os.environ["pipeline_name"] = str(DEFAULTS_PATH) component = TestEnvVarModel( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, ) assert component.name == str( diff --git a/tests/components/test_kafka_app.py b/tests/components/test_kafka_app.py index 2d2935c16..7f4d4fec3 100644 --- a/tests/components/test_kafka_app.py +++ b/tests/components/test_kafka_app.py @@ -30,8 +30,8 @@ def handlers(self) -> ComponentHandlers: def test_default_brokers(self, config: PipelineConfig, handlers: ComponentHandlers): kafka_app = KafkaApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "streams-app", "name": "example-name", @@ -52,8 +52,8 @@ def test_should_deploy_kafka_app( self, config: PipelineConfig, handlers: ComponentHandlers, mocker: MockerFixture ): kafka_app = KafkaApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "streams-app", "name": "example-name", diff --git a/tests/components/test_kafka_connect.py b/tests/components/test_kafka_connect.py index 9e192da7e..ccec219d2 100644 --- a/tests/components/test_kafka_connect.py +++ b/tests/components/test_kafka_connect.py @@ -56,8 +56,8 @@ def test_connector_config_parsing( topic_name = "connector-topic" connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(**{"topics": topic_name}), ) assert getattr(connector.app, "topics") == topic_name @@ -65,16 +65,16 @@ def test_connector_config_parsing( topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(**{"topics.regex": topic_pattern}), ) assert getattr(connector.app, "topics.regex") == topic_pattern connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -94,8 +94,8 @@ def test_from_section_parsing_input_topic( topic2 = "connector-topic2" connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), from_=FromSection( topics={ @@ -116,8 +116,8 @@ def test_from_section_parsing_input_pattern( topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), from_=FromSection( topics={topic_pattern: FromTopic(type=InputTopicTypes.INPUT_PATTERN)} @@ -133,8 +133,8 @@ def test_deploy_order( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -146,10 +146,10 @@ def test_deploy_order( ) mock_create_topics = mocker.patch.object( - connector.handlers.topic_handler, "create_topics" + connector._handlers.topic_handler, "create_topics" ) mock_create_connector = mocker.patch.object( - connector.handlers.connector_handler, "create_connector" + connector._handlers.connector_handler, "create_connector" ) mock = mocker.MagicMock() @@ -175,8 +175,8 @@ def test_destroy( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -188,7 +188,7 @@ def test_destroy( ) mock_destroy_connector = mocker.patch.object( - connector.handlers.connector_handler, "destroy_connector" + connector._handlers.connector_handler, "destroy_connector" ) connector.destroy(dry_run=True) @@ -206,8 +206,8 @@ def test_clean( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -219,10 +219,10 @@ def test_clean( ) mock_delete_topics = mocker.patch.object( - connector.handlers.topic_handler, "delete_topics" + connector._handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.patch.object( - connector.handlers.connector_handler, "clean_connector" + connector._handlers.connector_handler, "clean_connector" ) mock = mocker.MagicMock() @@ -251,8 +251,8 @@ def test_reset( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -264,10 +264,10 @@ def test_reset( ) mock_delete_topics = mocker.patch.object( - connector.handlers.topic_handler, "delete_topics" + connector._handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.patch.object( - connector.handlers.connector_handler, "clean_connector" + connector._handlers.connector_handler, "clean_connector" ) connector.reset(dry_run=True) @@ -289,16 +289,16 @@ def test_clean_without_to( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), ) mock_delete_topics = mocker.patch.object( - connector.handlers.topic_handler, "delete_topics" + connector._handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.patch.object( - connector.handlers.connector_handler, "clean_connector" + connector._handlers.connector_handler, "clean_connector" ) connector.clean(dry_run=True) @@ -320,8 +320,8 @@ def test_from_section_raises_exception( with pytest.raises(NotImplementedError): KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), from_=FromSection( topics={ @@ -338,8 +338,8 @@ def test_deploy_order( ): connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -351,11 +351,11 @@ def test_deploy_order( ) mock_create_topics = mocker.patch.object( - connector.handlers.topic_handler, "create_topics" + connector._handlers.topic_handler, "create_topics" ) mock_create_connector = mocker.patch.object( - connector.handlers.connector_handler, "create_connector" + connector._handlers.connector_handler, "create_connector" ) mock = mocker.MagicMock() @@ -384,8 +384,8 @@ def test_destroy( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -395,10 +395,10 @@ def test_destroy( } ), ) - assert connector.handlers.connector_handler + assert connector._handlers.connector_handler mock_destroy_connector = mocker.patch.object( - connector.handlers.connector_handler, "destroy_connector" + connector._handlers.connector_handler, "destroy_connector" ) connector.destroy(dry_run=True) @@ -419,8 +419,8 @@ def test_clean( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -430,13 +430,13 @@ def test_clean( } ), ) - assert connector.handlers.connector_handler + assert connector._handlers.connector_handler mock_delete_topics = mocker.patch.object( - connector.handlers.topic_handler, "delete_topics" + connector._handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.spy( - connector.handlers.connector_handler, "clean_connector" + connector._handlers.connector_handler, "clean_connector" ) mock = mocker.MagicMock() @@ -469,19 +469,19 @@ def test_clean_without_to( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), ) assert connector.to is None - assert connector.handlers.connector_handler + assert connector._handlers.connector_handler mock_delete_topics = mocker.patch.object( - connector.handlers.topic_handler, "delete_topics" + connector._handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.spy( - connector.handlers.connector_handler, "clean_connector" + connector._handlers.connector_handler, "clean_connector" ) connector.clean(dry_run=True) @@ -506,8 +506,8 @@ def test_reset( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -518,13 +518,13 @@ def test_reset( ), ) - assert connector.handlers.connector_handler + assert connector._handlers.connector_handler mock_delete_topics = mocker.patch.object( - connector.handlers.topic_handler, "delete_topics" + connector._handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.spy( - connector.handlers.connector_handler, "clean_connector" + connector._handlers.connector_handler, "clean_connector" ) connector.reset(dry_run=True) diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index dc2ad2bcf..a680ac0bf 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -59,9 +59,9 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( kubernetes_app = KubernetesApp( _type="test", - handlers=handlers, + _handlers=handlers, app=app_config, - config=config, + _config=config, name="test-kubernetes-apps", ) @@ -93,9 +93,9 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( kubernetes_app = KubernetesApp( _type="test", - handlers=handlers, + _handlers=handlers, app=app_config, - config=config, + _config=config, name="test-kubernetes-apps", version="3.4.5", ) @@ -145,9 +145,9 @@ def test_should_print_helm_diff_after_install_when_dry_run_and_helm_diff_enabled kubernetes_app = KubernetesApp( _type="test", - handlers=handlers, + _handlers=handlers, app=app_config, - config=config, + _config=config, name="test-kubernetes-apps", ) mocker.patch.object( @@ -166,9 +166,9 @@ def test_should_raise_not_implemented_error_when_helm_chart_is_not_set( kubernetes_app = KubernetesApp( _type="test", - handlers=handlers, + _handlers=handlers, app=app_config, - config=config, + _config=config, name="test-kubernetes-apps", ) @@ -190,9 +190,9 @@ def test_should_call_helm_uninstall_when_destroying_kubernetes_app( kubernetes_app = KubernetesApp( _type="test", - handlers=handlers, + _handlers=handlers, app=app_config, - config=config, + _config=config, name="test-kubernetes-apps", ) @@ -214,26 +214,26 @@ def test_should_raise_value_error_when_name_is_not_valid( assert KubernetesApp( _type="test", - handlers=handlers, + _handlers=handlers, app=app_config, - config=config, + _config=config, name="example-component-with-very-long-name-longer-than-most-of-our-kubernetes-apps", ) with pytest.raises(ValueError): assert KubernetesApp( _type="test", - handlers=handlers, + _handlers=handlers, app=app_config, - config=config, + _config=config, name="Not-Compatible*", ) with pytest.raises(ValueError): assert KubernetesApp( _type="test", - handlers=handlers, + _handlers=handlers, app=app_config, - config=config, + _config=config, name="snake_case", ) diff --git a/tests/components/test_pipeline_component.py b/tests/components/test_pipeline_component.py index 5decb00a5..a6d9e5f75 100644 --- a/tests/components/test_pipeline_component.py +++ b/tests/components/test_pipeline_component.py @@ -17,12 +17,12 @@ class TestPipelineComponent: def test_topic_substitution(self): pipeline_component = PipelineComponent( - handlers=ComponentHandlers( + _handlers=ComponentHandlers( schema_handler=MagicMock(), connector_handler=MagicMock(), topic_handler=MagicMock(), ), - config=pipeline_config.PipelineConfig( + _config=pipeline_config.PipelineConfig( defaults_path=DEFAULTS_PATH, environment="development", topic_name_config=TopicNameConfig( diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index f79cbc4a0..23ae60115 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -48,8 +48,8 @@ def producer_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> ProducerApp: return ProducerApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "producer-app", "name": self.PRODUCER_APP_NAME, @@ -71,8 +71,8 @@ def producer_app( def test_output_topics(self, config: PipelineConfig, handlers: ComponentHandlers): producer_app = ProducerApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "producer-app", "name": self.PRODUCER_APP_NAME, @@ -106,7 +106,7 @@ def test_deploy_order( mocker: MockerFixture, ): mock_create_topics = mocker.patch.object( - producer_app.handlers.topic_handler, "create_topics" + producer_app._handlers.topic_handler, "create_topics" ) mock_helm_upgrade_install = mocker.patch.object( diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 740e051df..5f5e6685a 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -49,8 +49,8 @@ def streams_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> StreamsApp: return StreamsApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -74,8 +74,8 @@ class AnotherType(StreamsApp): _type = "test" streams_app = AnotherType( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "test", "name": self.STREAMS_APP_NAME, @@ -114,8 +114,8 @@ def test_no_empty_input_topic( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "test", "name": self.STREAMS_APP_NAME, @@ -146,8 +146,8 @@ def test_no_empty_input_topic( def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandlers): with pytest.raises(ValueError): StreamsApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -167,8 +167,8 @@ def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandle with pytest.raises(ValueError): StreamsApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -184,8 +184,8 @@ def test_set_streams_output_from_to( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -226,8 +226,8 @@ def test_weave_inputs_from_prev_component( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -262,8 +262,8 @@ def test_deploy_order( mocker: MockerFixture, ): streams_app = StreamsApp( - handlers=handlers, - config=config, + _handlers=handlers, + _config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -295,7 +295,7 @@ def test_deploy_order( }, ) mock_create_topics = mocker.patch.object( - streams_app.handlers.topic_handler, "create_topics" + streams_app._handlers.topic_handler, "create_topics" ) mock_helm_upgrade_install = mocker.patch.object( streams_app.helm, "upgrade_install" diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 9f233cd9b..2b2db2db8 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -42,8 +42,8 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - handlers=self.handlers, - config=self.config, + _handlers=self._handlers, + _config=self._config, app={ "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 9b128bf1c..b011e24b0 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -26,8 +26,8 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - handlers=self.handlers, - config=self.config, + _handlers=self._handlers, + _config=self._config, app=KafkaConnectConfig( **{ "topics": topic_name, From a463ad740cb54337fc97a65f6dd0b2f0f76fb5ac Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 18 Jan 2023 17:45:29 +0100 Subject: [PATCH 02/44] Generate schema --- schema.json | 464 ++++++++++++++++++++++++++++++++++++++++++++++ scratch_schema.py | 33 ++++ 2 files changed, 497 insertions(+) create mode 100644 schema.json create mode 100644 scratch_schema.py diff --git a/schema.json b/schema.json new file mode 100644 index 000000000..9e738ec0d --- /dev/null +++ b/schema.json @@ -0,0 +1,464 @@ +{ + "title": "kpops pipeline schema", + "type": "array", + "items": { + "discriminator": { + "propertyName": "type", + "mapping": { + "streams-app": "#/definitions/StreamsApp", + "producer": "#/definitions/ProducerApp", + "kafka-source-connector": "#/definitions/KafkaSourceConnector", + "kafka-sink-connector": "#/definitions/KafkaSinkConnector" + } + }, + "oneOf": [ + { + "$ref": "#/definitions/StreamsApp" + }, + { + "$ref": "#/definitions/ProducerApp" + }, + { + "$ref": "#/definitions/KafkaSourceConnector" + }, + { + "$ref": "#/definitions/KafkaSinkConnector" + } + ] + }, + "definitions": { + "InputTopicTypes": { + "title": "InputTopicTypes", + "description": "An enumeration.", + "enum": [ + "input", + "extra", + "input-pattern", + "extra-pattern" + ], + "type": "string" + }, + "FromTopic": { + "title": "FromTopic", + "type": "object", + "properties": { + "type": { + "$ref": "#/definitions/InputTopicTypes" + }, + "role": { + "title": "Role", + "type": "string" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "FromSection": { + "title": "FromSection", + "type": "object", + "properties": { + "topics": { + "title": "Topics", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/FromTopic" + } + } + }, + "required": [ + "topics" + ], + "additionalProperties": false + }, + "StreamsConfig": { + "title": "StreamsConfig", + "description": "Streams Bootstrap streams section", + "type": "object", + "properties": { + "brokers": { + "title": "Brokers", + "type": "string" + }, + "schemaRegistryUrl": { + "title": "Schema Registry URL", + "type": "string" + }, + "inputTopics": { + "title": "Inputtopics", + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "inputPattern": { + "title": "Inputpattern", + "type": "string" + }, + "extraInputTopics": { + "title": "Extrainputtopics", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "extraInputPatterns": { + "title": "Extrainputpatterns", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "extraOutputTopics": { + "title": "Extraoutputtopics", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "outputTopic": { + "title": "Outputtopic", + "type": "string" + }, + "errorTopic": { + "title": "Errortopic", + "type": "string" + }, + "config": { + "title": "Config", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "brokers" + ] + }, + "StreamsAppAutoScaling": { + "title": "StreamsAppAutoScaling", + "type": "object", + "properties": { + "consumergroup": { + "title": "Consumergroup", + "type": "string" + }, + "topics": { + "title": "Topics", + "default": [], + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "consumergroup" + ] + }, + "StreamsAppConfig": { + "title": "StreamsAppConfig", + "description": "StreamsBoostrap app configurations.\n\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "type": "object", + "properties": { + "namespace": { + "title": "Namespace", + "type": "string" + }, + "streams": { + "$ref": "#/definitions/StreamsConfig" + }, + "nameOverride": { + "title": "Nameoverride", + "type": "string" + }, + "autoscaling": { + "$ref": "#/definitions/StreamsAppAutoScaling" + } + }, + "required": [ + "namespace", + "streams" + ] + }, + "OutputTopicTypes": { + "title": "OutputTopicTypes", + "description": "\n Types of output topic\n error (error topic), output (output topic), and extra topics. Every extra topic must have a role.\n ", + "enum": [ + "error", + "output", + "extra" + ], + "type": "string" + }, + "TopicConfig": { + "title": "TopicConfig", + "description": "Configures a topic", + "type": "object", + "properties": { + "type": { + "$ref": "#/definitions/OutputTopicTypes" + }, + "keySchema": { + "title": "Keyschema", + "type": "string" + }, + "valueSchema": { + "title": "Valueschema", + "type": "string" + }, + "partitions_count": { + "title": "Partitions Count", + "type": "integer" + }, + "replication_factor": { + "title": "Replication Factor", + "type": "integer" + }, + "configs": { + "title": "Configs", + "default": {}, + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + } + }, + "role": { + "title": "Role", + "type": "string" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "ToSection": { + "title": "ToSection", + "type": "object", + "properties": { + "models": { + "title": "Models", + "default": {}, + "type": "object" + }, + "topics": { + "title": "Topics", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/TopicConfig" + } + } + }, + "required": [ + "topics" + ] + }, + "StreamsApp": { + "title": "StreamsApp", + "description": "StreamsApp component that configures a streams bootstrap app", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "from": { + "$ref": "#/definitions/FromSection" + }, + "app": { + "$ref": "#/definitions/StreamsAppConfig" + }, + "to": { + "$ref": "#/definitions/ToSection" + }, + "version": { + "title": "Version", + "type": "string" + }, + "type": { + "title": "Type", + "default": "streams-app", + "enum": [ + "streams-app" + ], + "type": "string" + } + }, + "required": [ + "name", + "app" + ] + }, + "ProducerStreamsConfig": { + "title": "ProducerStreamsConfig", + "type": "object", + "properties": { + "brokers": { + "title": "Brokers", + "type": "string" + }, + "schemaRegistryUrl": { + "title": "Schema Registry URL", + "type": "string" + }, + "extraOutputTopics": { + "title": "Extraoutputtopics", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "outputTopic": { + "title": "Outputtopic", + "type": "string" + } + }, + "required": [ + "brokers" + ] + }, + "ProducerValues": { + "title": "ProducerValues", + "type": "object", + "properties": { + "namespace": { + "title": "Namespace", + "type": "string" + }, + "streams": { + "$ref": "#/definitions/ProducerStreamsConfig" + }, + "nameOverride": { + "title": "Nameoverride", + "type": "string" + } + }, + "required": [ + "namespace", + "streams" + ] + }, + "ProducerApp": { + "title": "ProducerApp", + "description": "Producer component\n\nThis producer holds configuration to use as values for the streams bootstrap produce helm chart.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "from": { + "$ref": "#/definitions/FromSection" + }, + "app": { + "$ref": "#/definitions/ProducerValues" + }, + "to": { + "$ref": "#/definitions/ToSection" + }, + "version": { + "title": "Version", + "type": "string" + }, + "type": { + "title": "Type", + "default": "producer", + "enum": [ + "producer" + ], + "type": "string" + } + }, + "required": [ + "name", + "app" + ] + }, + "KafkaConnectConfig": { + "title": "KafkaConnectConfig", + "type": "object", + "properties": {} + }, + "KafkaSourceConnector": { + "title": "KafkaSourceConnector", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "from": { + "$ref": "#/definitions/FromSection" + }, + "app": { + "$ref": "#/definitions/KafkaConnectConfig" + }, + "to": { + "$ref": "#/definitions/ToSection" + }, + "type": { + "title": "Type", + "default": "kafka-source-connector", + "enum": [ + "kafka-source-connector" + ], + "type": "string" + } + }, + "required": [ + "name", + "app" + ] + }, + "KafkaSinkConnector": { + "title": "KafkaSinkConnector", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "from": { + "$ref": "#/definitions/FromSection" + }, + "app": { + "$ref": "#/definitions/KafkaConnectConfig" + }, + "to": { + "$ref": "#/definitions/ToSection" + }, + "type": { + "title": "Type", + "default": "kafka-sink-connector", + "enum": [ + "kafka-sink-connector" + ], + "type": "string" + } + }, + "required": [ + "name", + "app" + ] + } + } +} diff --git a/scratch_schema.py b/scratch_schema.py new file mode 100644 index 000000000..d7fad83c0 --- /dev/null +++ b/scratch_schema.py @@ -0,0 +1,33 @@ +from typing import Annotated, Sequence + +from pydantic import Field, schema_json_of + +from kpops.components.base_components.kafka_connect import ( + KafkaSinkConnector, + KafkaSourceConnector, +) +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp + +# schema = schema_json_of(Pipeline, by_alias=True, indent=4) +# schema = schema_json_of( +# list[StreamsApp | ProducerApp | KafkaConnector], by_alias=True, indent=4 +# ) + +ComponentType = ( + StreamsApp + | ProducerApp + | KafkaSourceConnector + | KafkaSinkConnector + # | PipelineComponent +) + + +AnnotatedPipelineComponent = Annotated[ComponentType, Field(discriminator="type")] +schema = schema_json_of( + Sequence[AnnotatedPipelineComponent], + title="kpops pipeline schema", + by_alias=True, + indent=4, +) +print(schema) From 1322ff33ab7e7030580d5de4772da5395cdbc94c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 18 Jan 2023 18:06:44 +0100 Subject: [PATCH 03/44] WIP --- .../base_defaults_component.py | 11 ++++--- .../base_components/kafka_connect.py | 4 +-- .../producer/producer_app.py | 2 +- .../streams_bootstrap/streams/streams_app.py | 2 +- kpops/pipeline_generator/pipeline.py | 32 +++++++++---------- .../pipeline/snapshots/snap_test_pipeline.py | 26 +++++++-------- 6 files changed, 39 insertions(+), 38 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 15e917069..ffc730907 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -7,7 +7,7 @@ from typing import TypeVar import typer -from pydantic import BaseConfig, BaseModel, Field, PrivateAttr +from pydantic import BaseConfig, BaseModel, Field from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -19,11 +19,12 @@ class BaseDefaultsComponent(BaseModel): _type: str = Field(..., alias="type") - _enrich: bool = PrivateAttr(default=False) - _config: PipelineConfig = PrivateAttr(default=...) - _handlers: ComponentHandlers = PrivateAttr(default=...) + _enrich: bool = Field(default=False, alias="enrich", exclude=True) + _config: PipelineConfig = Field(default=..., alias="config", exclude=True) + _handlers: ComponentHandlers = Field(default=..., alias="handlers", exclude=True) class Config(BaseConfig): + # exclude = {"_enrich", "_config", "_handlers"} arbitrary_types_allowed = True def __init__(self, **kwargs): @@ -39,7 +40,7 @@ def extend_with_defaults(self, kwargs) -> dict: :return: enriched kwargs with tmp_defaults """ - config: PipelineConfig = kwargs["config"] + config: PipelineConfig = kwargs["_config"] log.debug( typer.style( "Enriching component of type ", fg=typer.colors.GREEN, bold=False diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 22c5bfaee..7699ab4c3 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -65,7 +65,7 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): _type = "kafka-source-connector" - type: Literal["kafka-source-connector"] = "kafka-source-connector" + dicriminator: Literal["kafka-source-connector"] = "kafka-source-connector" @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -92,7 +92,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): _type = "kafka-sink-connector" - type: Literal["kafka-sink-connector"] = "kafka-sink-connector" + dicriminator: Literal["kafka-sink-connector"] = "kafka-sink-connector" @override def add_input_topics(self, topics: list[str]) -> None: diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 71281685d..795ca87d4 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -21,7 +21,7 @@ class ProducerApp(KafkaApp): """ _type = "producer" - type: Literal["producer"] = "producer" + dicriminator: Literal["producer"] = "producer" app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 95f799fe8..639a92ec2 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -17,7 +17,7 @@ class StreamsApp(KafkaApp): """ _type = "streams-app" - type: Literal["streams-app"] = "streams-app" + dicriminator: Literal["streams-app"] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index e324b885e..2ccb910e3 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -145,24 +145,24 @@ def parse_components( env_components_index = create_env_components_index(environment_components) previous_component: PipelineComponent | None = None for component in component_list: + # try: try: - try: - component_type: str = component["type"] - except KeyError: - raise ValueError( - "Every component must have a type defined, this component does not have one." - ) - component_class = self.registry[component_type] - inflated_components = self.apply_component( - component, component_class, env_components_index, previous_component + component_type: str = component["type"] + except KeyError: + raise ValueError( + "Every component must have a type defined, this component does not have one." ) - self.populate_pipeline_component_names(inflated_components) - self.components.extend(inflated_components) - previous_component = inflated_components.pop() - except Exception as ex: - raise ParsingException( - f"Error enriching {component['type']} component {component['name']}" - ) from ex + component_class = self.registry[component_type] + inflated_components = self.apply_component( + component, component_class, env_components_index, previous_component + ) + self.populate_pipeline_component_names(inflated_components) + self.components.extend(inflated_components) + previous_component = inflated_components.pop() + # except Exception as ex: + # raise ParsingException( + # f"Error enriching {component['type']} component {component['name']}" + # ) from ex def populate_pipeline_component_names( self, inflated_components: list[PipelineComponent] diff --git a/tests/pipeline/snapshots/snap_test_pipeline.py b/tests/pipeline/snapshots/snap_test_pipeline.py index a7754e9a0..761930a30 100644 --- a/tests/pipeline/snapshots/snap_test_pipeline.py +++ b/tests/pipeline/snapshots/snap_test_pipeline.py @@ -115,7 +115,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-pipeline-with-inflate-converter-error", @@ -168,7 +168,7 @@ "resources": {"requests": {"memory": "3G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-pipeline-with-inflate-should-inflate-error", @@ -227,7 +227,7 @@ "namespace": "example-namespace", "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-kafka-connect-sink-streams-app-error", @@ -321,7 +321,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-first-pipeline-converter-error", @@ -374,7 +374,7 @@ "resources": {"requests": {"memory": "3G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error", @@ -416,7 +416,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-no-input-topic-pipeline-streams-app-error", @@ -448,7 +448,7 @@ "namespace": "example-namespace", "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-no-input-topic-pipeline-streams-app-error", @@ -497,7 +497,7 @@ "namespace": "example-namespace", "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-no-user-defined-components-streams-app-error", @@ -573,7 +573,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-pipeline-with-envs-converter-error", @@ -624,7 +624,7 @@ "resources": {"requests": {"memory": "3G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-pipeline-with-envs-filter-error", @@ -707,7 +707,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-component-type-substitution-converter-error", @@ -761,7 +761,7 @@ "resources": {"requests": {"memory": "3G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-component-type-substitution-filter-app-error", @@ -864,7 +864,7 @@ "namespace": "development-namespace", "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "config": { + "_config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-kafka-connect-sink-streams-app-development-error", From b2c948bd23d04e1105599462ccb9a85743a6c0af Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 10:09:57 +0100 Subject: [PATCH 04/44] Revert "WIP" This reverts commit 1322ff33ab7e7030580d5de4772da5395cdbc94c. --- .../base_defaults_component.py | 11 +++---- .../base_components/kafka_connect.py | 4 +-- .../producer/producer_app.py | 2 +- .../streams_bootstrap/streams/streams_app.py | 2 +- kpops/pipeline_generator/pipeline.py | 32 +++++++++---------- .../pipeline/snapshots/snap_test_pipeline.py | 26 +++++++-------- 6 files changed, 38 insertions(+), 39 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index ffc730907..15e917069 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -7,7 +7,7 @@ from typing import TypeVar import typer -from pydantic import BaseConfig, BaseModel, Field +from pydantic import BaseConfig, BaseModel, Field, PrivateAttr from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -19,12 +19,11 @@ class BaseDefaultsComponent(BaseModel): _type: str = Field(..., alias="type") - _enrich: bool = Field(default=False, alias="enrich", exclude=True) - _config: PipelineConfig = Field(default=..., alias="config", exclude=True) - _handlers: ComponentHandlers = Field(default=..., alias="handlers", exclude=True) + _enrich: bool = PrivateAttr(default=False) + _config: PipelineConfig = PrivateAttr(default=...) + _handlers: ComponentHandlers = PrivateAttr(default=...) class Config(BaseConfig): - # exclude = {"_enrich", "_config", "_handlers"} arbitrary_types_allowed = True def __init__(self, **kwargs): @@ -40,7 +39,7 @@ def extend_with_defaults(self, kwargs) -> dict: :return: enriched kwargs with tmp_defaults """ - config: PipelineConfig = kwargs["_config"] + config: PipelineConfig = kwargs["config"] log.debug( typer.style( "Enriching component of type ", fg=typer.colors.GREEN, bold=False diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 7699ab4c3..22c5bfaee 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -65,7 +65,7 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): _type = "kafka-source-connector" - dicriminator: Literal["kafka-source-connector"] = "kafka-source-connector" + type: Literal["kafka-source-connector"] = "kafka-source-connector" @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -92,7 +92,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): _type = "kafka-sink-connector" - dicriminator: Literal["kafka-sink-connector"] = "kafka-sink-connector" + type: Literal["kafka-sink-connector"] = "kafka-sink-connector" @override def add_input_topics(self, topics: list[str]) -> None: diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 795ca87d4..71281685d 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -21,7 +21,7 @@ class ProducerApp(KafkaApp): """ _type = "producer" - dicriminator: Literal["producer"] = "producer" + type: Literal["producer"] = "producer" app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 639a92ec2..95f799fe8 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -17,7 +17,7 @@ class StreamsApp(KafkaApp): """ _type = "streams-app" - dicriminator: Literal["streams-app"] = "streams-app" + type: Literal["streams-app"] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index 2ccb910e3..e324b885e 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -145,24 +145,24 @@ def parse_components( env_components_index = create_env_components_index(environment_components) previous_component: PipelineComponent | None = None for component in component_list: - # try: try: - component_type: str = component["type"] - except KeyError: - raise ValueError( - "Every component must have a type defined, this component does not have one." + try: + component_type: str = component["type"] + except KeyError: + raise ValueError( + "Every component must have a type defined, this component does not have one." + ) + component_class = self.registry[component_type] + inflated_components = self.apply_component( + component, component_class, env_components_index, previous_component ) - component_class = self.registry[component_type] - inflated_components = self.apply_component( - component, component_class, env_components_index, previous_component - ) - self.populate_pipeline_component_names(inflated_components) - self.components.extend(inflated_components) - previous_component = inflated_components.pop() - # except Exception as ex: - # raise ParsingException( - # f"Error enriching {component['type']} component {component['name']}" - # ) from ex + self.populate_pipeline_component_names(inflated_components) + self.components.extend(inflated_components) + previous_component = inflated_components.pop() + except Exception as ex: + raise ParsingException( + f"Error enriching {component['type']} component {component['name']}" + ) from ex def populate_pipeline_component_names( self, inflated_components: list[PipelineComponent] diff --git a/tests/pipeline/snapshots/snap_test_pipeline.py b/tests/pipeline/snapshots/snap_test_pipeline.py index 761930a30..a7754e9a0 100644 --- a/tests/pipeline/snapshots/snap_test_pipeline.py +++ b/tests/pipeline/snapshots/snap_test_pipeline.py @@ -115,7 +115,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-pipeline-with-inflate-converter-error", @@ -168,7 +168,7 @@ "resources": {"requests": {"memory": "3G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-pipeline-with-inflate-should-inflate-error", @@ -227,7 +227,7 @@ "namespace": "example-namespace", "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-kafka-connect-sink-streams-app-error", @@ -321,7 +321,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-first-pipeline-converter-error", @@ -374,7 +374,7 @@ "resources": {"requests": {"memory": "3G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error", @@ -416,7 +416,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-no-input-topic-pipeline-streams-app-error", @@ -448,7 +448,7 @@ "namespace": "example-namespace", "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-no-input-topic-pipeline-streams-app-error", @@ -497,7 +497,7 @@ "namespace": "example-namespace", "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-no-user-defined-components-streams-app-error", @@ -573,7 +573,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-pipeline-with-envs-converter-error", @@ -624,7 +624,7 @@ "resources": {"requests": {"memory": "3G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-pipeline-with-envs-filter-error", @@ -707,7 +707,7 @@ "resources": {"limits": {"memory": "2G"}, "requests": {"memory": "2G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-component-type-substitution-converter-error", @@ -761,7 +761,7 @@ "resources": {"requests": {"memory": "3G"}}, "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-component-type-substitution-filter-app-error", @@ -864,7 +864,7 @@ "namespace": "development-namespace", "streams": { "brokers": "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092", - "_config": { + "config": { "large.message.id.generator": "com.bakdata.kafka.MurmurHashIdGenerator" }, "errorTopic": "resources-kafka-connect-sink-streams-app-development-error", From 96b4a091912d4cc8eab745881e5ac86e795978d1 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 10:10:13 +0100 Subject: [PATCH 05/44] Revert "Refactor to make internal fields private" This reverts commit 9a8e3c0730f7c424dc8d2d5a80fa20ecbfe79dc5. --- .../base_defaults_component.py | 8 +- kpops/components/base_components/kafka_app.py | 14 +-- .../base_components/kafka_connect.py | 28 +++-- .../base_components/kubernetes_app.py | 4 +- .../base_components/models/to_section.py | 2 +- .../base_components/pipeline_component.py | 4 +- .../producer/producer_app.py | 11 +- .../streams_bootstrap/streams/streams_app.py | 11 +- kpops/pipeline_generator/pipeline.py | 8 +- .../test_base_defaults_component.py | 16 +-- tests/components/test_kafka_app.py | 8 +- tests/components/test_kafka_connect.py | 108 +++++++++--------- tests/components/test_kubernetes_app.py | 32 +++--- tests/components/test_pipeline_component.py | 4 +- tests/components/test_producer_app.py | 10 +- tests/components/test_streams_app.py | 34 +++--- tests/pipeline/test_components/components.py | 4 +- .../components.py | 4 +- 18 files changed, 151 insertions(+), 159 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 15e917069..bdf69d6d8 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -7,7 +7,7 @@ from typing import TypeVar import typer -from pydantic import BaseConfig, BaseModel, Field, PrivateAttr +from pydantic import BaseConfig, BaseModel, Field from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -19,9 +19,9 @@ class BaseDefaultsComponent(BaseModel): _type: str = Field(..., alias="type") - _enrich: bool = PrivateAttr(default=False) - _config: PipelineConfig = PrivateAttr(default=...) - _handlers: ComponentHandlers = PrivateAttr(default=...) + enrich: bool = Field(default=False, exclude=True) + config: PipelineConfig = Field(default=..., exclude=True) + handlers: ComponentHandlers = Field(default=..., exclude=True) class Config(BaseConfig): arbitrary_types_allowed = True diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index c9ba77926..c2fb849c5 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -2,7 +2,7 @@ import logging -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, Extra from typing_extensions import override from kpops.component_handlers.helm_wrapper.helm import Helm @@ -20,7 +20,7 @@ class KafkaStreamsConfig(BaseModel): brokers: str - schema_registry_url: str | None = Field(None, title="Schema Registry URL") + schema_registry_url: str | None = None class Config(CamelCaseConfig): extra = Extra.allow @@ -43,12 +43,12 @@ class KafkaApp(KubernetesApp): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self.app.streams.brokers = substitute( - self.app.streams.brokers, {"broker": self._config.broker} + self.app.streams.brokers, {"broker": self.config.broker} ) if self.app.streams.schema_registry_url: self.app.streams.schema_registry_url = substitute( self.app.streams.schema_registry_url, - {"schema_registry_url": self._config.schema_registry_url}, + {"schema_registry_url": self.config.schema_registry_url}, ) @property @@ -58,12 +58,12 @@ def clean_up_helm_chart(self) -> str: @override def deploy(self, dry_run: bool) -> None: if self.to: - self._handlers.topic_handler.create_topics( + self.handlers.topic_handler.create_topics( to_section=self.to, dry_run=dry_run ) - if self._handlers.schema_handler: - self._handlers.schema_handler.submit_schemas( + if self.handlers.schema_handler: + self.handlers.schema_handler.submit_schemas( to_section=self.to, dry_run=dry_run ) super().deploy(dry_run) diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 22c5bfaee..f126dce65 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -1,7 +1,7 @@ import json import os from abc import ABC -from typing import Literal, NoReturn +from typing import NoReturn from typing_extensions import override @@ -34,38 +34,37 @@ def prepare_connector_config(self) -> None: @override def deploy(self, dry_run: bool) -> None: if self.to: - self._handlers.topic_handler.create_topics( + self.handlers.topic_handler.create_topics( to_section=self.to, dry_run=dry_run ) - if self._handlers.schema_handler: - self._handlers.schema_handler.submit_schemas( + if self.handlers.schema_handler: + self.handlers.schema_handler.submit_schemas( to_section=self.to, dry_run=dry_run ) - self._handlers.connector_handler.create_connector( + self.handlers.connector_handler.create_connector( connector_name=self.name, kafka_connect_config=self.app, dry_run=dry_run ) @override def destroy(self, dry_run: bool) -> None: - self._handlers.connector_handler.destroy_connector( + self.handlers.connector_handler.destroy_connector( connector_name=self.name, dry_run=dry_run ) @override def clean(self, dry_run: bool) -> None: if self.to: - if self._handlers.schema_handler: - self._handlers.schema_handler.delete_schemas( + if self.handlers.schema_handler: + self.handlers.schema_handler.delete_schemas( to_section=self.to, dry_run=dry_run ) - self._handlers.topic_handler.delete_topics(self.to, dry_run=dry_run) + self.handlers.topic_handler.delete_topics(self.to, dry_run=dry_run) class KafkaSourceConnector(KafkaConnector): _type = "kafka-source-connector" - type: Literal["kafka-source-connector"] = "kafka-source-connector" @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -81,18 +80,17 @@ def clean(self, dry_run: bool) -> None: self.__run_kafka_connect_resetter(dry_run) def __run_kafka_connect_resetter(self, dry_run: bool) -> None: - self._handlers.connector_handler.clean_connector( + self.handlers.connector_handler.clean_connector( connector_name=self.name, connector_type=KafkaConnectorType.SOURCE, dry_run=dry_run, - retain_clean_jobs=self._config.retain_clean_jobs, + retain_clean_jobs=self.config.retain_clean_jobs, offset_topic=os.environ[f"{ENV_PREFIX}KAFKA_CONNECT_RESETTER_OFFSET_TOPIC"], ) class KafkaSinkConnector(KafkaConnector): _type = "kafka-sink-connector" - type: Literal["kafka-sink-connector"] = "kafka-sink-connector" @override def add_input_topics(self, topics: list[str]) -> None: @@ -121,10 +119,10 @@ def clean(self, dry_run: bool) -> None: def __clean_sink_connector( self, dry_run: bool, delete_consumer_group: bool ) -> None: - self._handlers.connector_handler.clean_connector( + self.handlers.connector_handler.clean_connector( connector_name=self.name, connector_type=KafkaConnectorType.SINK, dry_run=dry_run, - retain_clean_jobs=self._config.retain_clean_jobs, + retain_clean_jobs=self.config.retain_clean_jobs, delete_consumer_group=delete_consumer_group, ) diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 5250c34e5..451ea6b58 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -49,7 +49,7 @@ def __init__(self, **kwargs): @cached_property def helm(self) -> Helm: - helm = Helm(self._config.helm_config) + helm = Helm(self.config.helm_config) if self.helm_repo_config is not None: helm.add_repo( self.helm_repo_config.repository_name, @@ -60,7 +60,7 @@ def helm(self) -> Helm: @cached_property def helm_diff(self) -> HelmDiff: - return HelmDiff(self._config.helm_diff_config) + return HelmDiff(self.config.helm_diff_config) @property def helm_release_name(self) -> str: diff --git a/kpops/components/base_components/models/to_section.py b/kpops/components/base_components/models/to_section.py index 82502a660..c3be1caa5 100644 --- a/kpops/components/base_components/models/to_section.py +++ b/kpops/components/base_components/models/to_section.py @@ -23,7 +23,7 @@ class TopicConfig(BaseModel): value_schema: str | None = Field(default=None, alias="valueSchema") partitions_count: int | None = None replication_factor: int | None = None - configs: dict[str, str | int] = {} + configs: dict[str, str] = {} role: str | None = None class Config(BaseConfig): diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 6ef4ae71e..4003c6621 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -59,12 +59,12 @@ def substitute_component_variables(self, topic_name: str) -> str: :return: final topic name """ error_topic_name = self.substitute_component_names( - self._config.topic_name_config.default_error_topic_name, + self.config.topic_name_config.default_error_topic_name, self._type, **os.environ, ) output_topic_name = self.substitute_component_names( - self._config.topic_name_config.default_output_topic_name, + self.config.topic_name_config.default_output_topic_name, self._type, **os.environ, ) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 71281685d..9eaeaa427 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,5 +1,3 @@ -from typing import Literal - from pydantic import BaseConfig, Extra from typing_extensions import override @@ -21,7 +19,6 @@ class ProducerApp(KafkaApp): """ _type = "producer" - type: Literal["producer"] = "producer" app: ProducerValues class Config(BaseConfig): @@ -45,22 +42,22 @@ def add_extra_output_topic(self, topic_name: str, role: str) -> None: @override def get_helm_chart(self) -> str: - return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.PRODUCER_APP.value}" + return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.PRODUCER_APP.value}" @property @override def clean_up_helm_chart(self) -> str: - return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" + return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" @property @override def helm_repo_config(self) -> HelmRepoConfig | None: - return self._config.streams_bootstrap_helm_config + return self.config.streams_bootstrap_helm_config @override def clean(self, dry_run: bool) -> None: self._run_clean_up_job( values=self.to_helm_values(), dry_run=dry_run, - retain_clean_jobs=self._config.retain_clean_jobs, + retain_clean_jobs=self.config.retain_clean_jobs, ) diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 95f799fe8..b46957a20 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Literal - from pydantic import BaseConfig, Extra from typing_extensions import override @@ -17,7 +15,6 @@ class StreamsApp(KafkaApp): """ _type = "streams-app" - type: Literal["streams-app"] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): @@ -57,17 +54,17 @@ def add_extra_output_topic(self, topic_name: str, role: str) -> None: @override def get_helm_chart(self) -> str: - return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.STREAMS_APP.value}" + return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.STREAMS_APP.value}" @property @override def helm_repo_config(self) -> HelmRepoConfig | None: - return self._config.streams_bootstrap_helm_config + return self.config.streams_bootstrap_helm_config @property @override def clean_up_helm_chart(self) -> str: - return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" + return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" @override def reset(self, dry_run: bool) -> None: @@ -83,7 +80,7 @@ def __run_streams_clean_up_job(self, dry_run: bool, delete_output: bool) -> None self._run_clean_up_job( values=values, dry_run=dry_run, - retain_clean_jobs=self._config.retain_clean_jobs, + retain_clean_jobs=self.config.retain_clean_jobs, ) def __substitute_autoscaling_topic_names(self) -> None: diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index e324b885e..daa8be3f1 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -200,7 +200,7 @@ def enrich_component( env_components_index: dict[str, dict], ) -> PipelineComponent: component_object: PipelineComponent = component_class( - _handlers=self.handlers, _config=self.config, **component + handlers=self.handlers, config=self.config, **component ) env_component_definition = env_components_index.get(component_object.name, {}) pair = update_nested_pair( @@ -212,9 +212,9 @@ def enrich_component( component_object, pair ) return component_class( - _enrich=False, - _handlers=self.handlers, - _config=self.config, + enrich=False, + handlers=self.handlers, + config=self.config, **json_object, ) diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index 3e9341d64..51053206b 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -57,8 +57,8 @@ def test_inherit_defaults( self, config: PipelineConfig, handlers: ComponentHandlers ): component = TestChildModel( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, ) assert ( @@ -79,8 +79,8 @@ def test_inherit_defaults( def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): component = TestChildModel( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{"name": "name-defined-in-pipeline_generator"} ) @@ -104,8 +104,8 @@ def test_multiple_generations( self, config: PipelineConfig, handlers: ComponentHandlers ): component = TestGrandChildModel( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, ) assert ( @@ -134,8 +134,8 @@ class TestEnvVarModel(BaseDefaultsComponent): os.environ["pipeline_name"] = str(DEFAULTS_PATH) component = TestEnvVarModel( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, ) assert component.name == str( diff --git a/tests/components/test_kafka_app.py b/tests/components/test_kafka_app.py index 7f4d4fec3..2d2935c16 100644 --- a/tests/components/test_kafka_app.py +++ b/tests/components/test_kafka_app.py @@ -30,8 +30,8 @@ def handlers(self) -> ComponentHandlers: def test_default_brokers(self, config: PipelineConfig, handlers: ComponentHandlers): kafka_app = KafkaApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "streams-app", "name": "example-name", @@ -52,8 +52,8 @@ def test_should_deploy_kafka_app( self, config: PipelineConfig, handlers: ComponentHandlers, mocker: MockerFixture ): kafka_app = KafkaApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "streams-app", "name": "example-name", diff --git a/tests/components/test_kafka_connect.py b/tests/components/test_kafka_connect.py index ccec219d2..9e192da7e 100644 --- a/tests/components/test_kafka_connect.py +++ b/tests/components/test_kafka_connect.py @@ -56,8 +56,8 @@ def test_connector_config_parsing( topic_name = "connector-topic" connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(**{"topics": topic_name}), ) assert getattr(connector.app, "topics") == topic_name @@ -65,16 +65,16 @@ def test_connector_config_parsing( topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(**{"topics.regex": topic_pattern}), ) assert getattr(connector.app, "topics.regex") == topic_pattern connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -94,8 +94,8 @@ def test_from_section_parsing_input_topic( topic2 = "connector-topic2" connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), from_=FromSection( topics={ @@ -116,8 +116,8 @@ def test_from_section_parsing_input_pattern( topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), from_=FromSection( topics={topic_pattern: FromTopic(type=InputTopicTypes.INPUT_PATTERN)} @@ -133,8 +133,8 @@ def test_deploy_order( ): connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -146,10 +146,10 @@ def test_deploy_order( ) mock_create_topics = mocker.patch.object( - connector._handlers.topic_handler, "create_topics" + connector.handlers.topic_handler, "create_topics" ) mock_create_connector = mocker.patch.object( - connector._handlers.connector_handler, "create_connector" + connector.handlers.connector_handler, "create_connector" ) mock = mocker.MagicMock() @@ -175,8 +175,8 @@ def test_destroy( ): connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -188,7 +188,7 @@ def test_destroy( ) mock_destroy_connector = mocker.patch.object( - connector._handlers.connector_handler, "destroy_connector" + connector.handlers.connector_handler, "destroy_connector" ) connector.destroy(dry_run=True) @@ -206,8 +206,8 @@ def test_clean( ): connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -219,10 +219,10 @@ def test_clean( ) mock_delete_topics = mocker.patch.object( - connector._handlers.topic_handler, "delete_topics" + connector.handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.patch.object( - connector._handlers.connector_handler, "clean_connector" + connector.handlers.connector_handler, "clean_connector" ) mock = mocker.MagicMock() @@ -251,8 +251,8 @@ def test_reset( ): connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -264,10 +264,10 @@ def test_reset( ) mock_delete_topics = mocker.patch.object( - connector._handlers.topic_handler, "delete_topics" + connector.handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.patch.object( - connector._handlers.connector_handler, "clean_connector" + connector.handlers.connector_handler, "clean_connector" ) connector.reset(dry_run=True) @@ -289,16 +289,16 @@ def test_clean_without_to( ): connector = KafkaSinkConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), ) mock_delete_topics = mocker.patch.object( - connector._handlers.topic_handler, "delete_topics" + connector.handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.patch.object( - connector._handlers.connector_handler, "clean_connector" + connector.handlers.connector_handler, "clean_connector" ) connector.clean(dry_run=True) @@ -320,8 +320,8 @@ def test_from_section_raises_exception( with pytest.raises(NotImplementedError): KafkaSourceConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), from_=FromSection( topics={ @@ -338,8 +338,8 @@ def test_deploy_order( ): connector = KafkaSourceConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -351,11 +351,11 @@ def test_deploy_order( ) mock_create_topics = mocker.patch.object( - connector._handlers.topic_handler, "create_topics" + connector.handlers.topic_handler, "create_topics" ) mock_create_connector = mocker.patch.object( - connector._handlers.connector_handler, "create_connector" + connector.handlers.connector_handler, "create_connector" ) mock = mocker.MagicMock() @@ -384,8 +384,8 @@ def test_destroy( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -395,10 +395,10 @@ def test_destroy( } ), ) - assert connector._handlers.connector_handler + assert connector.handlers.connector_handler mock_destroy_connector = mocker.patch.object( - connector._handlers.connector_handler, "destroy_connector" + connector.handlers.connector_handler, "destroy_connector" ) connector.destroy(dry_run=True) @@ -419,8 +419,8 @@ def test_clean( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -430,13 +430,13 @@ def test_clean( } ), ) - assert connector._handlers.connector_handler + assert connector.handlers.connector_handler mock_delete_topics = mocker.patch.object( - connector._handlers.topic_handler, "delete_topics" + connector.handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.spy( - connector._handlers.connector_handler, "clean_connector" + connector.handlers.connector_handler, "clean_connector" ) mock = mocker.MagicMock() @@ -469,19 +469,19 @@ def test_clean_without_to( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), ) assert connector.to is None - assert connector._handlers.connector_handler + assert connector.handlers.connector_handler mock_delete_topics = mocker.patch.object( - connector._handlers.topic_handler, "delete_topics" + connector.handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.spy( - connector._handlers.connector_handler, "clean_connector" + connector.handlers.connector_handler, "clean_connector" ) connector.clean(dry_run=True) @@ -506,8 +506,8 @@ def test_reset( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -518,13 +518,13 @@ def test_reset( ), ) - assert connector._handlers.connector_handler + assert connector.handlers.connector_handler mock_delete_topics = mocker.patch.object( - connector._handlers.topic_handler, "delete_topics" + connector.handlers.topic_handler, "delete_topics" ) mock_clean_connector = mocker.spy( - connector._handlers.connector_handler, "clean_connector" + connector.handlers.connector_handler, "clean_connector" ) connector.reset(dry_run=True) diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index a680ac0bf..dc2ad2bcf 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -59,9 +59,9 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( kubernetes_app = KubernetesApp( _type="test", - _handlers=handlers, + handlers=handlers, app=app_config, - _config=config, + config=config, name="test-kubernetes-apps", ) @@ -93,9 +93,9 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( kubernetes_app = KubernetesApp( _type="test", - _handlers=handlers, + handlers=handlers, app=app_config, - _config=config, + config=config, name="test-kubernetes-apps", version="3.4.5", ) @@ -145,9 +145,9 @@ def test_should_print_helm_diff_after_install_when_dry_run_and_helm_diff_enabled kubernetes_app = KubernetesApp( _type="test", - _handlers=handlers, + handlers=handlers, app=app_config, - _config=config, + config=config, name="test-kubernetes-apps", ) mocker.patch.object( @@ -166,9 +166,9 @@ def test_should_raise_not_implemented_error_when_helm_chart_is_not_set( kubernetes_app = KubernetesApp( _type="test", - _handlers=handlers, + handlers=handlers, app=app_config, - _config=config, + config=config, name="test-kubernetes-apps", ) @@ -190,9 +190,9 @@ def test_should_call_helm_uninstall_when_destroying_kubernetes_app( kubernetes_app = KubernetesApp( _type="test", - _handlers=handlers, + handlers=handlers, app=app_config, - _config=config, + config=config, name="test-kubernetes-apps", ) @@ -214,26 +214,26 @@ def test_should_raise_value_error_when_name_is_not_valid( assert KubernetesApp( _type="test", - _handlers=handlers, + handlers=handlers, app=app_config, - _config=config, + config=config, name="example-component-with-very-long-name-longer-than-most-of-our-kubernetes-apps", ) with pytest.raises(ValueError): assert KubernetesApp( _type="test", - _handlers=handlers, + handlers=handlers, app=app_config, - _config=config, + config=config, name="Not-Compatible*", ) with pytest.raises(ValueError): assert KubernetesApp( _type="test", - _handlers=handlers, + handlers=handlers, app=app_config, - _config=config, + config=config, name="snake_case", ) diff --git a/tests/components/test_pipeline_component.py b/tests/components/test_pipeline_component.py index a6d9e5f75..5decb00a5 100644 --- a/tests/components/test_pipeline_component.py +++ b/tests/components/test_pipeline_component.py @@ -17,12 +17,12 @@ class TestPipelineComponent: def test_topic_substitution(self): pipeline_component = PipelineComponent( - _handlers=ComponentHandlers( + handlers=ComponentHandlers( schema_handler=MagicMock(), connector_handler=MagicMock(), topic_handler=MagicMock(), ), - _config=pipeline_config.PipelineConfig( + config=pipeline_config.PipelineConfig( defaults_path=DEFAULTS_PATH, environment="development", topic_name_config=TopicNameConfig( diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 23ae60115..f79cbc4a0 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -48,8 +48,8 @@ def producer_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> ProducerApp: return ProducerApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "producer-app", "name": self.PRODUCER_APP_NAME, @@ -71,8 +71,8 @@ def producer_app( def test_output_topics(self, config: PipelineConfig, handlers: ComponentHandlers): producer_app = ProducerApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "producer-app", "name": self.PRODUCER_APP_NAME, @@ -106,7 +106,7 @@ def test_deploy_order( mocker: MockerFixture, ): mock_create_topics = mocker.patch.object( - producer_app._handlers.topic_handler, "create_topics" + producer_app.handlers.topic_handler, "create_topics" ) mock_helm_upgrade_install = mocker.patch.object( diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 5f5e6685a..740e051df 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -49,8 +49,8 @@ def streams_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> StreamsApp: return StreamsApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -74,8 +74,8 @@ class AnotherType(StreamsApp): _type = "test" streams_app = AnotherType( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "test", "name": self.STREAMS_APP_NAME, @@ -114,8 +114,8 @@ def test_no_empty_input_topic( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "test", "name": self.STREAMS_APP_NAME, @@ -146,8 +146,8 @@ def test_no_empty_input_topic( def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandlers): with pytest.raises(ValueError): StreamsApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -167,8 +167,8 @@ def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandle with pytest.raises(ValueError): StreamsApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -184,8 +184,8 @@ def test_set_streams_output_from_to( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -226,8 +226,8 @@ def test_weave_inputs_from_prev_component( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -262,8 +262,8 @@ def test_deploy_order( mocker: MockerFixture, ): streams_app = StreamsApp( - _handlers=handlers, - _config=config, + handlers=handlers, + config=config, **{ "type": "streams-app", "name": self.STREAMS_APP_NAME, @@ -295,7 +295,7 @@ def test_deploy_order( }, ) mock_create_topics = mocker.patch.object( - streams_app._handlers.topic_handler, "create_topics" + streams_app.handlers.topic_handler, "create_topics" ) mock_helm_upgrade_install = mocker.patch.object( streams_app.helm, "upgrade_install" diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 2b2db2db8..9f233cd9b 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -42,8 +42,8 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - _handlers=self._handlers, - _config=self._config, + handlers=self.handlers, + config=self.config, app={ "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index b011e24b0..9b128bf1c 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -26,8 +26,8 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - _handlers=self._handlers, - _config=self._config, + handlers=self.handlers, + config=self.config, app=KafkaConnectConfig( **{ "topics": topic_name, From 2c30f482b69a3d45c4483e6b312ae94ec5f27988 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 11:31:02 +0100 Subject: [PATCH 06/44] Fix usage of private attr --- .../base_defaults_component.py | 11 +- .../producer/producer_app.py | 3 + .../streams_bootstrap/streams/streams_app.py | 3 + kpops/pipeline_generator/pipeline.py | 1 + schema.json | 527 +++++++++++++++--- scratch_schema.py | 17 +- 6 files changed, 455 insertions(+), 107 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index bdf69d6d8..75205f568 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -7,7 +7,7 @@ from typing import TypeVar import typer -from pydantic import BaseConfig, BaseModel, Field +from pydantic import BaseConfig, BaseModel, Field, PrivateAttr from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -21,16 +21,21 @@ class BaseDefaultsComponent(BaseModel): enrich: bool = Field(default=False, exclude=True) config: PipelineConfig = Field(default=..., exclude=True) - handlers: ComponentHandlers = Field(default=..., exclude=True) + _handlers: ComponentHandlers = PrivateAttr() class Config(BaseConfig): arbitrary_types_allowed = True - def __init__(self, **kwargs): + def __init__(self, **kwargs) -> None: + self._handlers = kwargs["handlers"] if kwargs.get("enrich", True): kwargs = self.extend_with_defaults(kwargs) super().__init__(**kwargs) + @property + def handlers(self) -> ComponentHandlers: + return self._handlers + def extend_with_defaults(self, kwargs) -> dict: """ Merges tmp_defaults with all tmp_defaults for parent classes diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 9eaeaa427..fd98c4d16 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import BaseConfig, Extra from typing_extensions import override @@ -19,6 +21,7 @@ class ProducerApp(KafkaApp): """ _type = "producer" + discriminator: Literal["producer"] = "producer" app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index b46957a20..4b88ee946 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Literal + from pydantic import BaseConfig, Extra from typing_extensions import override @@ -15,6 +17,7 @@ class StreamsApp(KafkaApp): """ _type = "streams-app" + discriminator: Literal["streams-app"] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index daa8be3f1..d44175470 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -223,6 +223,7 @@ def substitute_component_specific_variables( component_object: PipelineComponent, pair: dict ) -> dict: # Override component config with component config in pipeline environment definition + pair.pop("handlers") json_object: dict = json.loads( substitute( json.dumps(pair), diff --git a/schema.json b/schema.json index 9e738ec0d..deca3287f 100644 --- a/schema.json +++ b/schema.json @@ -3,12 +3,10 @@ "type": "array", "items": { "discriminator": { - "propertyName": "type", + "propertyName": "discriminator", "mapping": { "streams-app": "#/definitions/StreamsApp", - "producer": "#/definitions/ProducerApp", - "kafka-source-connector": "#/definitions/KafkaSourceConnector", - "kafka-sink-connector": "#/definitions/KafkaSinkConnector" + "producer": "#/definitions/ProducerApp" } }, "oneOf": [ @@ -17,16 +15,417 @@ }, { "$ref": "#/definitions/ProducerApp" - }, - { - "$ref": "#/definitions/KafkaSourceConnector" - }, - { - "$ref": "#/definitions/KafkaSinkConnector" } ] }, "definitions": { + "TopicNameConfig": { + "title": "TopicNameConfig", + "description": "Base class for settings, allowing values to be overridden by environment variables.\n\nThis is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),\nHeroku and any 12 factor app design.", + "type": "object", + "properties": { + "default_output_topic_name": { + "title": "Default Output Topic Name", + "description": "Configures the value for the variable ${output_topic_name}", + "default": "${pipeline_name}-${component_name}", + "env_names": [ + "default_output_topic_name" + ], + "type": "string" + }, + "default_error_topic_name": { + "title": "Default Error Topic Name", + "description": "Configures the value for the variable ${error_topic_name}", + "default": "${pipeline_name}-${component_name}-error", + "env_names": [ + "default_error_topic_name" + ], + "type": "string" + } + }, + "additionalProperties": false + }, + "HelmConfig": { + "title": "HelmConfig", + "type": "object", + "properties": { + "context": { + "title": "Context", + "env_names": [ + "context" + ], + "type": "string" + }, + "debug": { + "title": "Debug", + "default": false, + "env_names": [ + "debug" + ], + "type": "boolean" + } + }, + "additionalProperties": false + }, + "HelmDiffConfig": { + "title": "HelmDiffConfig", + "type": "object", + "properties": { + "enable": { + "title": "Enable", + "default": false, + "env_names": [ + "enable" + ], + "type": "boolean" + }, + "ignore": { + "title": "Ignore", + "env_names": [ + "ignore" + ], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "additionalProperties": false + }, + "RepoAuthFlags": { + "title": "RepoAuthFlags", + "type": "object", + "properties": { + "username": { + "title": "Username", + "env_names": [ + "username" + ], + "type": "string" + }, + "password": { + "title": "Password", + "env_names": [ + "password" + ], + "type": "string" + }, + "ca_file": { + "title": "Ca File", + "env_names": [ + "ca_file" + ], + "type": "string", + "format": "path" + }, + "insecure_skip_tls_verify": { + "title": "Insecure Skip Tls Verify", + "default": false, + "env_names": [ + "insecure_skip_tls_verify" + ], + "type": "boolean" + } + }, + "additionalProperties": false + }, + "HelmRepoConfig": { + "title": "HelmRepoConfig", + "type": "object", + "properties": { + "repository_name": { + "title": "Repository Name", + "env_names": [ + "repository_name" + ], + "type": "string" + }, + "url": { + "title": "Url", + "env_names": [ + "url" + ], + "type": "string" + }, + "repo_auth_flags": { + "title": "Repo Auth Flags", + "env_names": [ + "repo_auth_flags" + ], + "allOf": [ + { + "$ref": "#/definitions/RepoAuthFlags" + } + ] + } + }, + "required": [ + "repository_name", + "url" + ], + "additionalProperties": false + }, + "KafkaConnectResetterHelmConfig": { + "title": "KafkaConnectResetterHelmConfig", + "description": "Base class for settings, allowing values to be overridden by environment variables.\n\nThis is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),\nHeroku and any 12 factor app design.", + "type": "object", + "properties": { + "helm_config": { + "title": "Helm Config", + "description": "Configuration of Kafka connect resetter Helm Chart", + "default": { + "repository_name": "bakdata-kafka-connect-resetter", + "url": "https://bakdata.github.io/kafka-connect-resetter/", + "repo_auth_flags": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + } + }, + "env_names": [ + "helm_config" + ], + "allOf": [ + { + "$ref": "#/definitions/HelmRepoConfig" + } + ] + }, + "version": { + "title": "Version", + "default": "1.0.4", + "env_names": [ + "version" + ], + "type": "string" + }, + "helm_values": { + "title": "Helm Values", + "description": "Overriding Kafka Connect Resetter Helm values. E.g. to override the Image Tag etc.", + "default": {}, + "env_names": [ + "helm_values" + ], + "type": "object" + }, + "namespace": { + "title": "Namespace", + "default": "", + "env_names": [ + "namespace" + ], + "type": "string" + } + }, + "additionalProperties": false + }, + "PipelineConfig": { + "title": "PipelineConfig", + "description": "Base class for settings, allowing values to be overridden by environment variables.\n\nThis is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),\nHeroku and any 12 factor app design.", + "type": "object", + "properties": { + "defaults_path": { + "title": "Defaults Path", + "description": "The path to the folder containing the defaults file and the environment defaults files.", + "env_names": [ + "defaults_path" + ], + "type": "string", + "format": "path" + }, + "environment": { + "title": "Environment", + "description": "The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).", + "env": "KPOPS_ENVIRONMENT", + "example": "development", + "env_names": [ + "kpops_environment" + ], + "type": "string" + }, + "broker": { + "title": "Broker", + "description": "The kafka broker address.", + "env": "KPOPS_KAFKA_BROKER", + "env_names": [ + "kpops_kafka_broker" + ], + "type": "string" + }, + "defaults_filename_prefix": { + "title": "Defaults Filename Prefix", + "description": "The name of the defaults file and the prefix of the defaults environment file.", + "default": "defaults", + "env_names": [ + "defaults_filename_prefix" + ], + "type": "string" + }, + "topic_name_config": { + "title": "Topic Name Config", + "description": "Configure the topic name variables you can use in the pipeline definition.", + "default": { + "default_output_topic_name": "${pipeline_name}-${component_name}", + "default_error_topic_name": "${pipeline_name}-${component_name}-error" + }, + "env_names": [ + "topic_name_config" + ], + "allOf": [ + { + "$ref": "#/definitions/TopicNameConfig" + } + ] + }, + "kafka_rest_host": { + "title": "Kafka Rest Host", + "description": "Address to the rest proxy REST API.", + "env": "KPOPS_REST_PROXY_HOST", + "example": "http://localhost:8082", + "env_names": [ + "kpops_rest_proxy_host" + ], + "type": "string" + }, + "kafka_connect_host": { + "title": "Kafka Connect Host", + "description": "Address to the kafka connect REST API.", + "env": "KPOPS_CONNECT_HOST", + "example": "http://localhost:8083", + "env_names": [ + "kpops_connect_host" + ], + "type": "string" + }, + "timeout": { + "title": "Timeout", + "description": "The timeout in seconds that specifies when actions like deletion or deploy timeout.", + "default": 300, + "env": "KPOPS_TIMEOUT", + "env_names": [ + "kpops_timeout" + ], + "type": "integer" + }, + "pipeline_prefix": { + "title": "Pipeline Prefix", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "default": "${pipeline_name}-", + "env": "KPOPS_PIPELINE_PREFIX", + "env_names": [ + "kpops_pipeline_prefix" + ], + "type": "string" + }, + "helm_config": { + "title": "Helm Config", + "default": { + "context": null, + "debug": false + }, + "env_names": [ + "helm_config" + ], + "allOf": [ + { + "$ref": "#/definitions/HelmConfig" + } + ] + }, + "helm_diff_config": { + "title": "Helm Diff Config", + "default": { + "enable": false, + "ignore": [] + }, + "env_names": [ + "helm_diff_config" + ], + "allOf": [ + { + "$ref": "#/definitions/HelmDiffConfig" + } + ] + }, + "streams_bootstrap_helm_config": { + "title": "Streams Bootstrap Helm Config", + "description": "Configuration for Streams Bootstrap Helm Charts", + "default": { + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/", + "repo_auth_flags": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + } + }, + "env_names": [ + "streams_bootstrap_helm_config" + ], + "allOf": [ + { + "$ref": "#/definitions/HelmRepoConfig" + } + ] + }, + "kafka_connect_resetter_config": { + "title": "Kafka Connect Resetter Config", + "description": "Configuration of kafka connect resetter helm chart and values. This is used for cleaning/resettting Kafka connectors, see https://github.com/bakdata/kafka-connect-resetter", + "default": { + "helm_config": { + "repository_name": "bakdata-kafka-connect-resetter", + "url": "https://bakdata.github.io/kafka-connect-resetter/", + "repo_auth_flags": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + } + }, + "version": "1.0.4", + "helm_values": {}, + "namespace": "" + }, + "env_names": [ + "kafka_connect_resetter_config" + ], + "allOf": [ + { + "$ref": "#/definitions/KafkaConnectResetterHelmConfig" + } + ] + }, + "retain_clean_jobs": { + "title": "Retain Clean Jobs", + "description": "Whether to retain clean up jobs in the cluster or uninstall the, after completion.", + "default": false, + "env": "KPOPS_RETAIN_CLEAN_JOBS", + "env_names": [ + "kpops_retain_clean_jobs" + ], + "type": "boolean" + }, + "schema_registry_url": { + "title": "Schema Registry Url", + "description": "The URL to schema registry.", + "example": "http://localhost:8081", + "env": "KPOPS_SCHEMA_REGISTRY_URL", + "env_names": [ + "kpops_schema_registry_url" + ], + "type": "string" + } + }, + "required": [ + "defaults_path", + "environment", + "broker" + ], + "additionalProperties": false + }, "InputTopicTypes": { "title": "InputTopicTypes", "description": "An enumeration.", @@ -82,7 +481,7 @@ "type": "string" }, "schemaRegistryUrl": { - "title": "Schema Registry URL", + "title": "Schemaregistryurl", "type": "string" }, "inputTopics": { @@ -193,7 +592,7 @@ }, "OutputTopicTypes": { "title": "OutputTopicTypes", - "description": "\n Types of output topic\n error (error topic), output (output topic), and extra topics. Every extra topic must have a role.\n ", + "description": "Types of output topic\nerror (error topic), output (output topic), and extra topics. Every extra topic must have a role.", "enum": [ "error", "output", @@ -230,14 +629,7 @@ "default": {}, "type": "object", "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] + "type": "string" } }, "role": { @@ -276,6 +668,14 @@ "description": "StreamsApp component that configures a streams bootstrap app", "type": "object", "properties": { + "enrich": { + "title": "Enrich", + "default": false, + "type": "boolean" + }, + "config": { + "$ref": "#/definitions/PipelineConfig" + }, "name": { "title": "Name", "type": "string" @@ -293,8 +693,8 @@ "title": "Version", "type": "string" }, - "type": { - "title": "Type", + "discriminator": { + "title": "Discriminator", "default": "streams-app", "enum": [ "streams-app" @@ -303,6 +703,7 @@ } }, "required": [ + "config", "name", "app" ] @@ -316,7 +717,7 @@ "type": "string" }, "schemaRegistryUrl": { - "title": "Schema Registry URL", + "title": "Schemaregistryurl", "type": "string" }, "extraOutputTopics": { @@ -362,6 +763,14 @@ "description": "Producer component\n\nThis producer holds configuration to use as values for the streams bootstrap produce helm chart.", "type": "object", "properties": { + "enrich": { + "title": "Enrich", + "default": false, + "type": "boolean" + }, + "config": { + "$ref": "#/definitions/PipelineConfig" + }, "name": { "title": "Name", "type": "string" @@ -379,8 +788,8 @@ "title": "Version", "type": "string" }, - "type": { - "title": "Type", + "discriminator": { + "title": "Discriminator", "default": "producer", "enum": [ "producer" @@ -389,73 +798,7 @@ } }, "required": [ - "name", - "app" - ] - }, - "KafkaConnectConfig": { - "title": "KafkaConnectConfig", - "type": "object", - "properties": {} - }, - "KafkaSourceConnector": { - "title": "KafkaSourceConnector", - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "from": { - "$ref": "#/definitions/FromSection" - }, - "app": { - "$ref": "#/definitions/KafkaConnectConfig" - }, - "to": { - "$ref": "#/definitions/ToSection" - }, - "type": { - "title": "Type", - "default": "kafka-source-connector", - "enum": [ - "kafka-source-connector" - ], - "type": "string" - } - }, - "required": [ - "name", - "app" - ] - }, - "KafkaSinkConnector": { - "title": "KafkaSinkConnector", - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "from": { - "$ref": "#/definitions/FromSection" - }, - "app": { - "$ref": "#/definitions/KafkaConnectConfig" - }, - "to": { - "$ref": "#/definitions/ToSection" - }, - "type": { - "title": "Type", - "default": "kafka-sink-connector", - "enum": [ - "kafka-sink-connector" - ], - "type": "string" - } - }, - "required": [ + "config", "name", "app" ] diff --git a/scratch_schema.py b/scratch_schema.py index d7fad83c0..6af385cd8 100644 --- a/scratch_schema.py +++ b/scratch_schema.py @@ -2,28 +2,21 @@ from pydantic import Field, schema_json_of -from kpops.components.base_components.kafka_connect import ( - KafkaSinkConnector, - KafkaSourceConnector, -) from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp -# schema = schema_json_of(Pipeline, by_alias=True, indent=4) -# schema = schema_json_of( -# list[StreamsApp | ProducerApp | KafkaConnector], by_alias=True, indent=4 -# ) - ComponentType = ( StreamsApp | ProducerApp - | KafkaSourceConnector - | KafkaSinkConnector + # | KafkaSourceConnector + # | KafkaSinkConnector # | PipelineComponent ) -AnnotatedPipelineComponent = Annotated[ComponentType, Field(discriminator="type")] +AnnotatedPipelineComponent = Annotated[ + ComponentType, Field(discriminator="discriminator") +] schema = schema_json_of( Sequence[AnnotatedPipelineComponent], title="kpops pipeline schema", From 448cc686f92bc1e658ed3dd1c7d5341ebbaa6ae4 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 12:24:14 +0100 Subject: [PATCH 07/44] Fix static class variable --- kpops/cli/registry.py | 2 +- .../base_defaults_component.py | 10 +- kpops/components/base_components/kafka_app.py | 3 +- .../base_components/kafka_connect.py | 8 +- .../base_components/kubernetes_app.py | 3 +- .../base_components/pipeline_component.py | 8 +- .../producer/producer_app.py | 5 +- .../streams_bootstrap/streams/streams_app.py | 5 +- kpops/pipeline_generator/pipeline.py | 2 +- schema.json | 113 +++++++++++++++++- scratch_schema.py | 12 +- tests/cli/test_registry.py | 14 ++- .../test_base_defaults_component.py | 27 ++--- tests/components/test_kafka_connect.py | 20 ++-- tests/components/test_kubernetes_app.py | 16 +-- tests/components/test_pipeline_component.py | 2 +- tests/components/test_streams_app.py | 5 +- tests/pipeline/test_components/components.py | 10 +- .../components.py | 8 +- 19 files changed, 189 insertions(+), 84 deletions(-) diff --git a/kpops/cli/registry.py b/kpops/cli/registry.py index ee94570b9..de5e14f8a 100644 --- a/kpops/cli/registry.py +++ b/kpops/cli/registry.py @@ -29,7 +29,7 @@ def find_components(self, module_name: str) -> None: :param module_name: name of the python module """ for _class in _find_classes(module_name, PipelineComponent): - self._classes[_class._type] = _class + self._classes[_class.type] = _class def __getitem__(self, component_type: str) -> type[PipelineComponent]: try: diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 75205f568..c47a36a52 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -4,7 +4,7 @@ from collections import deque from collections.abc import Mapping, Sequence from pathlib import Path -from typing import TypeVar +from typing import ClassVar, TypeVar import typer from pydantic import BaseConfig, BaseModel, Field, PrivateAttr @@ -17,7 +17,7 @@ class BaseDefaultsComponent(BaseModel): - _type: str = Field(..., alias="type") + type: ClassVar[str] = Field(..., const=True) # component type discriminator enrich: bool = Field(default=False, exclude=True) config: PipelineConfig = Field(default=..., exclude=True) @@ -65,13 +65,13 @@ def extend_with_defaults(self, kwargs) -> dict: if not environment_default_file_path.exists(): kwargs = update_nested( kwargs, - defaults_from_yaml(main_default_file_path, base._type), + defaults_from_yaml(main_default_file_path, base.type), ) else: kwargs = update_nested( kwargs, - defaults_from_yaml(environment_default_file_path, base._type), - defaults_from_yaml(main_default_file_path, base._type), + defaults_from_yaml(environment_default_file_path, base.type), + defaults_from_yaml(main_default_file_path, base.type), ) return kwargs diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index c2fb849c5..341cd0622 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from typing import ClassVar, Literal from pydantic import BaseModel, Extra from typing_extensions import override @@ -37,7 +38,7 @@ class KafkaApp(KubernetesApp): Producer or streaming apps should inherit from this class. """ - _type = "kafka-app" + type: ClassVar[Literal["kafka-app"]] = "kafka-app" app: KafkaAppConfig def __init__(self, **kwargs) -> None: diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index f126dce65..8545d0c82 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -1,7 +1,7 @@ import json import os from abc import ABC -from typing import NoReturn +from typing import ClassVar, Literal, NoReturn from typing_extensions import override @@ -14,7 +14,7 @@ class KafkaConnector(PipelineComponent, ABC): - _type = "kafka-connect" + type: ClassVar[Literal["kafka-connect"]] = "kafka-connect" app: KafkaConnectConfig def __init__(self, **kwargs) -> None: @@ -64,7 +64,7 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): - _type = "kafka-source-connector" + type: ClassVar[Literal["kafka-source-connector"]] = "kafka-source-connector" @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -90,7 +90,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): - _type = "kafka-sink-connector" + type: ClassVar[Literal["kafka-sink-connector"]] = "kafka-sink-connector" @override def add_input_topics(self, topics: list[str]) -> None: diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 451ea6b58..112d2edb9 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -3,6 +3,7 @@ import logging import re from functools import cached_property +from typing import ClassVar, Literal from pydantic import BaseModel from typing_extensions import override @@ -35,7 +36,7 @@ class Config(CamelCaseConfig): class KubernetesApp(PipelineComponent): """Base kubernetes app""" - _type = "kubernetes-app" + type: ClassVar[Literal["kubernetes-app"]] = "kubernetes-app" app: KubernetesAppConfig version: str | None = None diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 4003c6621..9ac1367a2 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -60,17 +60,17 @@ def substitute_component_variables(self, topic_name: str) -> str: """ error_topic_name = self.substitute_component_names( self.config.topic_name_config.default_error_topic_name, - self._type, + self.type, **os.environ, ) output_topic_name = self.substitute_component_names( self.config.topic_name_config.default_output_topic_name, - self._type, + self.type, **os.environ, ) return self.substitute_component_names( topic_name, - self._type, + self.type, error_topic_name=error_topic_name, output_topic_name=output_topic_name, ) @@ -147,7 +147,7 @@ def weave_from_topics(self, prev_component_to: ToSection) -> None: def substitute_name(self): if self.name: - self.name = self.substitute_component_names(self.name, self._type) + self.name = self.substitute_component_names(self.name, self.type) else: raise ValueError("Every component must have a name in the end.") diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index fd98c4d16..ee72d1801 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import ClassVar, Literal from pydantic import BaseConfig, Extra from typing_extensions import override @@ -20,8 +20,7 @@ class ProducerApp(KafkaApp): This producer holds configuration to use as values for the streams bootstrap produce helm chart. """ - _type = "producer" - discriminator: Literal["producer"] = "producer" + type: ClassVar[Literal["producer"]] = "producer" app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 4b88ee946..cd631161e 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal +from typing import ClassVar, Literal from pydantic import BaseConfig, Extra from typing_extensions import override @@ -16,8 +16,7 @@ class StreamsApp(KafkaApp): StreamsApp component that configures a streams bootstrap app """ - _type = "streams-app" - discriminator: Literal["streams-app"] = "streams-app" + type: ClassVar[Literal["streams-app"]] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index d44175470..ae5bf74b7 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -228,7 +228,7 @@ def substitute_component_specific_variables( substitute( json.dumps(pair), { - "component_type": component_object._type, + "component_type": component_object.type, "component_name": component_object.name, }, ) diff --git a/schema.json b/schema.json index deca3287f..f73ef7a1f 100644 --- a/schema.json +++ b/schema.json @@ -3,10 +3,12 @@ "type": "array", "items": { "discriminator": { - "propertyName": "discriminator", + "propertyName": "type", "mapping": { "streams-app": "#/definitions/StreamsApp", - "producer": "#/definitions/ProducerApp" + "producer": "#/definitions/ProducerApp", + "kafka-source-connector": "#/definitions/KafkaSourceConnector", + "kafka-sink-connector": "#/definitions/KafkaSinkConnector" } }, "oneOf": [ @@ -15,6 +17,12 @@ }, { "$ref": "#/definitions/ProducerApp" + }, + { + "$ref": "#/definitions/KafkaSourceConnector" + }, + { + "$ref": "#/definitions/KafkaSinkConnector" } ] }, @@ -668,6 +676,14 @@ "description": "StreamsApp component that configures a streams bootstrap app", "type": "object", "properties": { + "type": { + "title": "Type", + "default": "streams-app", + "enum": [ + "streams-app" + ], + "type": "string" + }, "enrich": { "title": "Enrich", "default": false, @@ -763,6 +779,14 @@ "description": "Producer component\n\nThis producer holds configuration to use as values for the streams bootstrap produce helm chart.", "type": "object", "properties": { + "type": { + "title": "Type", + "default": "producer", + "enum": [ + "producer" + ], + "type": "string" + }, "enrich": { "title": "Enrich", "default": false, @@ -787,14 +811,91 @@ "version": { "title": "Version", "type": "string" + } + }, + "required": [ + "config", + "name", + "app" + ] + }, + "KafkaConnectConfig": { + "title": "KafkaConnectConfig", + "type": "object", + "properties": {} + }, + "KafkaSourceConnector": { + "title": "KafkaSourceConnector", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "kafka-source-connector", + "enum": [ + "kafka-source-connector" + ], + "type": "string" }, - "discriminator": { - "title": "Discriminator", - "default": "producer", + "enrich": { + "title": "Enrich", + "default": false, + "type": "boolean" + }, + "config": { + "$ref": "#/definitions/PipelineConfig" + }, + "name": { + "title": "Name", + "type": "string" + }, + "from": { + "$ref": "#/definitions/FromSection" + }, + "app": { + "$ref": "#/definitions/KafkaConnectConfig" + }, + "to": { + "$ref": "#/definitions/ToSection" + } + }, + "required": [ + "config", + "name", + "app" + ] + }, + "KafkaSinkConnector": { + "title": "KafkaSinkConnector", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "kafka-sink-connector", "enum": [ - "producer" + "kafka-sink-connector" ], "type": "string" + }, + "enrich": { + "title": "Enrich", + "default": false, + "type": "boolean" + }, + "config": { + "$ref": "#/definitions/PipelineConfig" + }, + "name": { + "title": "Name", + "type": "string" + }, + "from": { + "$ref": "#/definitions/FromSection" + }, + "app": { + "$ref": "#/definitions/KafkaConnectConfig" + }, + "to": { + "$ref": "#/definitions/ToSection" } }, "required": [ diff --git a/scratch_schema.py b/scratch_schema.py index 6af385cd8..30c16a1c5 100644 --- a/scratch_schema.py +++ b/scratch_schema.py @@ -2,21 +2,23 @@ from pydantic import Field, schema_json_of +from kpops.components.base_components.kafka_connect import ( + KafkaSinkConnector, + KafkaSourceConnector, +) from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp ComponentType = ( StreamsApp | ProducerApp - # | KafkaSourceConnector - # | KafkaSinkConnector + | KafkaSourceConnector + | KafkaSinkConnector # | PipelineComponent ) -AnnotatedPipelineComponent = Annotated[ - ComponentType, Field(discriminator="discriminator") -] +AnnotatedPipelineComponent = Annotated[ComponentType, Field(discriminator="type")] schema = schema_json_of( Sequence[AnnotatedPipelineComponent], title="kpops pipeline schema", diff --git a/tests/cli/test_registry.py b/tests/cli/test_registry.py index 1135b40f1..619613b77 100644 --- a/tests/cli/test_registry.py +++ b/tests/cli/test_registry.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import ClassVar, Literal + import pytest from pydantic import BaseModel @@ -8,12 +10,12 @@ class SubComponent(PipelineComponent): - _type: str = "sub_component" + type: ClassVar[Literal["sub-component"]] = "sub-component" pass class SubSubComponent(SubComponent): - _type: str = "sub_sub_component" + type: ClassVar[Literal["sub-sub-component"]] = "sub-sub-component" pass @@ -44,10 +46,10 @@ def test_registry(): assert registry._classes == {} registry.find_components(MODULE) assert registry._classes == { - SubComponent._type: SubComponent, - SubSubComponent._type: SubSubComponent, + SubComponent.type: SubComponent, + SubSubComponent.type: SubSubComponent, } - assert registry[SubComponent._type] == SubComponent - assert registry[SubSubComponent._type] == SubSubComponent + assert registry[SubComponent.type] == SubComponent + assert registry[SubSubComponent.type] == SubSubComponent with pytest.raises(ClassNotFoundError): registry["doesnt-exist"] diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index 51053206b..31b4d4a07 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -1,5 +1,6 @@ import os from pathlib import Path +from typing import ClassVar, Literal from unittest.mock import MagicMock import pytest @@ -16,7 +17,7 @@ class TestParentModel(BaseDefaultsComponent): __test__ = False - _type: str = "parent" + type: ClassVar[Literal["parent"]] = "parent" name: str | None = None value: float | None = None hard_coded: str = "hard_coded_value" @@ -24,14 +25,14 @@ class TestParentModel(BaseDefaultsComponent): class TestChildModel(TestParentModel): __test__ = False - _type: str = "child" + type: ClassVar[Literal["child"]] = "child" nice: dict | None = None another_hard_coded: str = "another_hard_coded_value" class TestGrandChildModel(TestChildModel): __test__ = False - _type: str = "grand-child" + type: ClassVar[Literal["grand-child"]] = "grand-child" grand_child: str | None = None @@ -56,10 +57,7 @@ class TestBaseDefaultsComponent: def test_inherit_defaults( self, config: PipelineConfig, handlers: ComponentHandlers ): - component = TestChildModel( - handlers=handlers, - config=config, - ) + component = TestChildModel(config=config, **{"handlers": handlers}) assert ( component.name == "fake-child-name" @@ -79,9 +77,11 @@ def test_inherit_defaults( def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): component = TestChildModel( - handlers=handlers, config=config, - **{"name": "name-defined-in-pipeline_generator"} + **{ + "name": "name-defined-in-pipeline_generator", + "handlers": handlers, + }, ) assert ( @@ -103,10 +103,7 @@ def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): def test_multiple_generations( self, config: PipelineConfig, handlers: ComponentHandlers ): - component = TestGrandChildModel( - handlers=handlers, - config=config, - ) + component = TestGrandChildModel(config=config, **{"handlers": handlers}) assert ( component.name == "fake-child-name" @@ -129,13 +126,13 @@ def test_env_var_substitution( self, config: PipelineConfig, handlers: ComponentHandlers ): class TestEnvVarModel(BaseDefaultsComponent): - _type: str = "env-var-test" + type: ClassVar[Literal["env-var-test"]] = "env-var-test" name: str | None = None os.environ["pipeline_name"] = str(DEFAULTS_PATH) component = TestEnvVarModel( - handlers=handlers, config=config, + **{"handlers": handlers}, ) assert component.name == str( diff --git a/tests/components/test_kafka_connect.py b/tests/components/test_kafka_connect.py index 9e192da7e..c7e6916cc 100644 --- a/tests/components/test_kafka_connect.py +++ b/tests/components/test_kafka_connect.py @@ -56,24 +56,23 @@ def test_connector_config_parsing( topic_name = "connector-topic" connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(**{"topics": topic_name}), + **{"handlers": handlers}, ) assert getattr(connector.app, "topics") == topic_name topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(**{"topics.regex": topic_pattern}), + **{"handlers": handlers}, ) assert getattr(connector.app, "topics.regex") == topic_pattern connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(), to=ToSection( @@ -81,6 +80,7 @@ def test_connector_config_parsing( "${error_topic_name}": TopicConfig(type=OutputTopicTypes.ERROR), } ), + **{"handlers": handlers}, ) assert ( getattr(connector.app, "errors.deadletterqueue.topic.name") @@ -94,7 +94,6 @@ def test_from_section_parsing_input_topic( topic2 = "connector-topic2" connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(), from_=FromSection( @@ -103,6 +102,7 @@ def test_from_section_parsing_input_topic( topic2: FromTopic(type=InputTopicTypes.INPUT), } ), + **{"handlers": handlers}, ) assert getattr(connector.app, "topics") == f"{topic1},{topic2}" @@ -116,12 +116,12 @@ def test_from_section_parsing_input_pattern( topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(), from_=FromSection( topics={topic_pattern: FromTopic(type=InputTopicTypes.INPUT_PATTERN)} ), + **{"handlers": handlers}, ) assert getattr(connector.app, "topics.regex") == topic_pattern @@ -133,7 +133,6 @@ def test_deploy_order( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(), to=ToSection( @@ -143,6 +142,7 @@ def test_deploy_order( ), } ), + **{"handlers": handlers}, ) mock_create_topics = mocker.patch.object( @@ -175,7 +175,6 @@ def test_destroy( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(), to=ToSection( @@ -185,6 +184,7 @@ def test_destroy( ), } ), + **{"handlers": handlers}, ) mock_destroy_connector = mocker.patch.object( @@ -206,7 +206,6 @@ def test_clean( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(), to=ToSection( @@ -216,6 +215,7 @@ def test_clean( ), } ), + **{"handlers": handlers}, ) mock_delete_topics = mocker.patch.object( @@ -251,7 +251,6 @@ def test_reset( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(), to=ToSection( @@ -261,6 +260,7 @@ def test_reset( ), } ), + **{"handlers": handlers}, ) mock_delete_topics = mocker.patch.object( @@ -289,9 +289,9 @@ def test_clean_without_to( ): connector = KafkaSinkConnector( name="test-connector", - handlers=handlers, config=config, app=KafkaConnectConfig(), + **{"handlers": handlers}, ) mock_delete_topics = mocker.patch.object( diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index dc2ad2bcf..3d8455566 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -58,7 +58,7 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( app_config = KubernetesAppConfig(namespace="test-namespace") kubernetes_app = KubernetesApp( - _type="test", + type="test", handlers=handlers, app=app_config, config=config, @@ -92,7 +92,7 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( app_config = KubernetesAppConfig(namespace="test-namespace") kubernetes_app = KubernetesApp( - _type="test", + type="test", handlers=handlers, app=app_config, config=config, @@ -144,7 +144,7 @@ def test_should_print_helm_diff_after_install_when_dry_run_and_helm_diff_enabled app_config = KubernetesAppConfig(namespace="test-namespace") kubernetes_app = KubernetesApp( - _type="test", + type="test", handlers=handlers, app=app_config, config=config, @@ -165,7 +165,7 @@ def test_should_raise_not_implemented_error_when_helm_chart_is_not_set( app_config = KubernetesAppConfig(namespace="test-namespace") kubernetes_app = KubernetesApp( - _type="test", + type="test", handlers=handlers, app=app_config, config=config, @@ -189,7 +189,7 @@ def test_should_call_helm_uninstall_when_destroying_kubernetes_app( app_config = KubernetesAppConfig(namespace="test-namespace") kubernetes_app = KubernetesApp( - _type="test", + type="test", handlers=handlers, app=app_config, config=config, @@ -213,7 +213,7 @@ def test_should_raise_value_error_when_name_is_not_valid( app_config = KubernetesAppConfig(namespace="test") assert KubernetesApp( - _type="test", + type="test", handlers=handlers, app=app_config, config=config, @@ -222,7 +222,7 @@ def test_should_raise_value_error_when_name_is_not_valid( with pytest.raises(ValueError): assert KubernetesApp( - _type="test", + type="test", handlers=handlers, app=app_config, config=config, @@ -231,7 +231,7 @@ def test_should_raise_value_error_when_name_is_not_valid( with pytest.raises(ValueError): assert KubernetesApp( - _type="test", + type="test", handlers=handlers, app=app_config, config=config, diff --git a/tests/components/test_pipeline_component.py b/tests/components/test_pipeline_component.py index 5decb00a5..a20a940f8 100644 --- a/tests/components/test_pipeline_component.py +++ b/tests/components/test_pipeline_component.py @@ -31,7 +31,7 @@ def test_topic_substitution(self): ), ), name="test-pipeline-component", - _type="plane-pipeline-component", + type="plane-pipeline-component", to=ToSection( models={}, topics={ diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 740e051df..10e204e42 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import ClassVar, Literal from unittest.mock import MagicMock import pytest @@ -71,14 +72,14 @@ def streams_app( def test_set_topics(self, config: PipelineConfig, handlers: ComponentHandlers): class AnotherType(StreamsApp): - _type = "test" + type: ClassVar[Literal["test"]] = "test" streams_app = AnotherType( - handlers=handlers, config=config, **{ "type": "test", "name": self.STREAMS_APP_NAME, + "handlers": handlers, "app": { "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 9f233cd9b..de492ec51 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, ClassVar, Literal from schema_registry.client.schema import AvroSchema from typing_extensions import override @@ -14,11 +14,11 @@ class ImportProducer(ProducerApp): - _type = "scheduled-producer" + type: ClassVar[Literal["scheduled-producer"]] = "scheduled-producer" class Converter(StreamsApp): - _type = "converter" + type: ClassVar[Literal["converter"]] = "converter" class SubStreamsApp(StreamsApp): @@ -28,11 +28,11 @@ class SubStreamsApp(StreamsApp): class Filter(SubStreamsApp): """Subsubclass of StreamsApp to test inheritance.""" - _type = "filter" + type: ClassVar[Literal["filter"]] = "filter" class InflateStep(StreamsApp): - _type = "should-inflate" + type: ClassVar[Literal["should-inflate"]] = "should-inflate" @override def inflate(self) -> list[PipelineComponent]: diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 9b128bf1c..cf0e13e3a 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -1,3 +1,5 @@ +from typing import ClassVar, Literal + from typing_extensions import override from kpops.component_handlers.kafka_connect.model import KafkaConnectConfig @@ -8,15 +10,15 @@ class ImportProducer(ProducerApp): - _type = "scheduled-producer" + type: ClassVar[Literal["scheduled-producer"]] = "scheduled-producer" class Converter(StreamsApp): - _type = "converter" + type: ClassVar[Literal["converter"]] = "converter" class InflateStep(StreamsApp): - _type = "should-inflate" + type: ClassVar[Literal["should-inflate"]] = "should-inflate" @override def inflate(self) -> list[PipelineComponent]: From e5d5c518af53b159ec6f1678bdfb88171170425a Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 13:01:28 +0100 Subject: [PATCH 08/44] Remove class var --- .../base_components/base_defaults_component.py | 4 ++-- kpops/components/base_components/kafka_app.py | 4 ++-- kpops/components/base_components/kafka_connect.py | 8 ++++---- kpops/components/base_components/kubernetes_app.py | 4 ++-- .../streams_bootstrap/producer/producer_app.py | 4 ++-- .../streams_bootstrap/streams/streams_app.py | 4 ++-- tests/cli/test_registry.py | 6 +++--- tests/components/test_base_defaults_component.py | 10 +++++----- tests/components/test_streams_app.py | 4 ++-- tests/pipeline/test_components/components.py | 10 +++++----- .../components.py | 8 ++++---- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index c47a36a52..4b0e567c4 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -4,7 +4,7 @@ from collections import deque from collections.abc import Mapping, Sequence from pathlib import Path -from typing import ClassVar, TypeVar +from typing import TypeVar import typer from pydantic import BaseConfig, BaseModel, Field, PrivateAttr @@ -17,7 +17,7 @@ class BaseDefaultsComponent(BaseModel): - type: ClassVar[str] = Field(..., const=True) # component type discriminator + type: str = Field(..., const=True) # component type discriminator enrich: bool = Field(default=False, exclude=True) config: PipelineConfig = Field(default=..., exclude=True) diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 341cd0622..942fc8a1d 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import ClassVar, Literal +from typing import Literal from pydantic import BaseModel, Extra from typing_extensions import override @@ -38,7 +38,7 @@ class KafkaApp(KubernetesApp): Producer or streaming apps should inherit from this class. """ - type: ClassVar[Literal["kafka-app"]] = "kafka-app" + type: Literal["kafka-app"] = "kafka-app" app: KafkaAppConfig def __init__(self, **kwargs) -> None: diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 8545d0c82..edb368b64 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -1,7 +1,7 @@ import json import os from abc import ABC -from typing import ClassVar, Literal, NoReturn +from typing import Literal, NoReturn from typing_extensions import override @@ -14,7 +14,7 @@ class KafkaConnector(PipelineComponent, ABC): - type: ClassVar[Literal["kafka-connect"]] = "kafka-connect" + type: Literal["kafka-connect"] = "kafka-connect" app: KafkaConnectConfig def __init__(self, **kwargs) -> None: @@ -64,7 +64,7 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): - type: ClassVar[Literal["kafka-source-connector"]] = "kafka-source-connector" + type: Literal["kafka-source-connector"] = "kafka-source-connector" @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -90,7 +90,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): - type: ClassVar[Literal["kafka-sink-connector"]] = "kafka-sink-connector" + type: Literal["kafka-sink-connector"] = "kafka-sink-connector" @override def add_input_topics(self, topics: list[str]) -> None: diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 112d2edb9..8f8e0e676 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -3,7 +3,7 @@ import logging import re from functools import cached_property -from typing import ClassVar, Literal +from typing import Literal from pydantic import BaseModel from typing_extensions import override @@ -36,7 +36,7 @@ class Config(CamelCaseConfig): class KubernetesApp(PipelineComponent): """Base kubernetes app""" - type: ClassVar[Literal["kubernetes-app"]] = "kubernetes-app" + type: Literal["kubernetes-app"] = "kubernetes-app" app: KubernetesAppConfig version: str | None = None diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index ee72d1801..ded790012 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,4 +1,4 @@ -from typing import ClassVar, Literal +from typing import Literal from pydantic import BaseConfig, Extra from typing_extensions import override @@ -20,7 +20,7 @@ class ProducerApp(KafkaApp): This producer holds configuration to use as values for the streams bootstrap produce helm chart. """ - type: ClassVar[Literal["producer"]] = "producer" + type: Literal["producer"] = "producer" app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index cd631161e..56a64353a 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import ClassVar, Literal +from typing import Literal from pydantic import BaseConfig, Extra from typing_extensions import override @@ -16,7 +16,7 @@ class StreamsApp(KafkaApp): StreamsApp component that configures a streams bootstrap app """ - type: ClassVar[Literal["streams-app"]] = "streams-app" + type: Literal["streams-app"] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): diff --git a/tests/cli/test_registry.py b/tests/cli/test_registry.py index 619613b77..f1058df8a 100644 --- a/tests/cli/test_registry.py +++ b/tests/cli/test_registry.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import ClassVar, Literal +from typing import Literal import pytest from pydantic import BaseModel @@ -10,12 +10,12 @@ class SubComponent(PipelineComponent): - type: ClassVar[Literal["sub-component"]] = "sub-component" + type: Literal["sub-component"] = "sub-component" pass class SubSubComponent(SubComponent): - type: ClassVar[Literal["sub-sub-component"]] = "sub-sub-component" + type: Literal["sub-sub-component"] = "sub-sub-component" pass diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index 31b4d4a07..2c37f9b71 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import ClassVar, Literal +from typing import Literal from unittest.mock import MagicMock import pytest @@ -17,7 +17,7 @@ class TestParentModel(BaseDefaultsComponent): __test__ = False - type: ClassVar[Literal["parent"]] = "parent" + type: Literal["parent"] = "parent" name: str | None = None value: float | None = None hard_coded: str = "hard_coded_value" @@ -25,14 +25,14 @@ class TestParentModel(BaseDefaultsComponent): class TestChildModel(TestParentModel): __test__ = False - type: ClassVar[Literal["child"]] = "child" + type: Literal["child"] = "child" nice: dict | None = None another_hard_coded: str = "another_hard_coded_value" class TestGrandChildModel(TestChildModel): __test__ = False - type: ClassVar[Literal["grand-child"]] = "grand-child" + type: Literal["grand-child"] = "grand-child" grand_child: str | None = None @@ -126,7 +126,7 @@ def test_env_var_substitution( self, config: PipelineConfig, handlers: ComponentHandlers ): class TestEnvVarModel(BaseDefaultsComponent): - type: ClassVar[Literal["env-var-test"]] = "env-var-test" + type: Literal["env-var-test"] = "env-var-test" name: str | None = None os.environ["pipeline_name"] = str(DEFAULTS_PATH) diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 10e204e42..59fabe370 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import ClassVar, Literal +from typing import Literal from unittest.mock import MagicMock import pytest @@ -72,7 +72,7 @@ def streams_app( def test_set_topics(self, config: PipelineConfig, handlers: ComponentHandlers): class AnotherType(StreamsApp): - type: ClassVar[Literal["test"]] = "test" + type: Literal["test"] = "test" streams_app = AnotherType( config=config, diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index de492ec51..b140e0805 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -1,4 +1,4 @@ -from typing import Any, ClassVar, Literal +from typing import Any, Literal from schema_registry.client.schema import AvroSchema from typing_extensions import override @@ -14,11 +14,11 @@ class ImportProducer(ProducerApp): - type: ClassVar[Literal["scheduled-producer"]] = "scheduled-producer" + type: Literal["scheduled-producer"] = "scheduled-producer" class Converter(StreamsApp): - type: ClassVar[Literal["converter"]] = "converter" + type: Literal["converter"] = "converter" class SubStreamsApp(StreamsApp): @@ -28,11 +28,11 @@ class SubStreamsApp(StreamsApp): class Filter(SubStreamsApp): """Subsubclass of StreamsApp to test inheritance.""" - type: ClassVar[Literal["filter"]] = "filter" + type: Literal["filter"] = "filter" class InflateStep(StreamsApp): - type: ClassVar[Literal["should-inflate"]] = "should-inflate" + type: Literal["should-inflate"] = "should-inflate" @override def inflate(self) -> list[PipelineComponent]: diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index cf0e13e3a..d7e217c50 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -1,4 +1,4 @@ -from typing import ClassVar, Literal +from typing import Literal from typing_extensions import override @@ -10,15 +10,15 @@ class ImportProducer(ProducerApp): - type: ClassVar[Literal["scheduled-producer"]] = "scheduled-producer" + type: Literal["scheduled-producer"] = "scheduled-producer" class Converter(StreamsApp): - type: ClassVar[Literal["converter"]] = "converter" + type: Literal["converter"] = "converter" class InflateStep(StreamsApp): - type: ClassVar[Literal["should-inflate"]] = "should-inflate" + type: Literal["should-inflate"] = "should-inflate" @override def inflate(self) -> list[PipelineComponent]: From dd6730a75a9524b8b517b7b7d64842d885fcb12a Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 13:10:24 +0100 Subject: [PATCH 09/44] Fix test --- schema.json | 8 -------- tests/cli/test_registry.py | 8 ++++---- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/schema.json b/schema.json index f73ef7a1f..d67ae1e34 100644 --- a/schema.json +++ b/schema.json @@ -708,14 +708,6 @@ "version": { "title": "Version", "type": "string" - }, - "discriminator": { - "title": "Discriminator", - "default": "streams-app", - "enum": [ - "streams-app" - ], - "type": "string" } }, "required": [ diff --git a/tests/cli/test_registry.py b/tests/cli/test_registry.py index f1058df8a..334f16ac2 100644 --- a/tests/cli/test_registry.py +++ b/tests/cli/test_registry.py @@ -46,10 +46,10 @@ def test_registry(): assert registry._classes == {} registry.find_components(MODULE) assert registry._classes == { - SubComponent.type: SubComponent, - SubSubComponent.type: SubSubComponent, + "sub-component": SubComponent, + "sub-sub-component": SubSubComponent, } - assert registry[SubComponent.type] == SubComponent - assert registry[SubSubComponent.type] == SubSubComponent + assert registry["sub-component"] == SubComponent + assert registry["sub-sub-component"] == SubSubComponent with pytest.raises(ClassNotFoundError): registry["doesnt-exist"] From 455fb8fde6d6f645b44e7e51f502d6ccef32934f Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 13:10:37 +0100 Subject: [PATCH 10/44] Revert "Remove class var" This reverts commit e5d5c518af53b159ec6f1678bdfb88171170425a. --- .../base_components/base_defaults_component.py | 4 ++-- kpops/components/base_components/kafka_app.py | 4 ++-- kpops/components/base_components/kafka_connect.py | 8 ++++---- kpops/components/base_components/kubernetes_app.py | 4 ++-- .../streams_bootstrap/producer/producer_app.py | 4 ++-- .../streams_bootstrap/streams/streams_app.py | 4 ++-- tests/cli/test_registry.py | 6 +++--- tests/components/test_base_defaults_component.py | 10 +++++----- tests/components/test_streams_app.py | 4 ++-- tests/pipeline/test_components/components.py | 10 +++++----- .../components.py | 8 ++++---- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 4b0e567c4..c47a36a52 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -4,7 +4,7 @@ from collections import deque from collections.abc import Mapping, Sequence from pathlib import Path -from typing import TypeVar +from typing import ClassVar, TypeVar import typer from pydantic import BaseConfig, BaseModel, Field, PrivateAttr @@ -17,7 +17,7 @@ class BaseDefaultsComponent(BaseModel): - type: str = Field(..., const=True) # component type discriminator + type: ClassVar[str] = Field(..., const=True) # component type discriminator enrich: bool = Field(default=False, exclude=True) config: PipelineConfig = Field(default=..., exclude=True) diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 942fc8a1d..341cd0622 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Literal +from typing import ClassVar, Literal from pydantic import BaseModel, Extra from typing_extensions import override @@ -38,7 +38,7 @@ class KafkaApp(KubernetesApp): Producer or streaming apps should inherit from this class. """ - type: Literal["kafka-app"] = "kafka-app" + type: ClassVar[Literal["kafka-app"]] = "kafka-app" app: KafkaAppConfig def __init__(self, **kwargs) -> None: diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index edb368b64..8545d0c82 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -1,7 +1,7 @@ import json import os from abc import ABC -from typing import Literal, NoReturn +from typing import ClassVar, Literal, NoReturn from typing_extensions import override @@ -14,7 +14,7 @@ class KafkaConnector(PipelineComponent, ABC): - type: Literal["kafka-connect"] = "kafka-connect" + type: ClassVar[Literal["kafka-connect"]] = "kafka-connect" app: KafkaConnectConfig def __init__(self, **kwargs) -> None: @@ -64,7 +64,7 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): - type: Literal["kafka-source-connector"] = "kafka-source-connector" + type: ClassVar[Literal["kafka-source-connector"]] = "kafka-source-connector" @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -90,7 +90,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): - type: Literal["kafka-sink-connector"] = "kafka-sink-connector" + type: ClassVar[Literal["kafka-sink-connector"]] = "kafka-sink-connector" @override def add_input_topics(self, topics: list[str]) -> None: diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 8f8e0e676..112d2edb9 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -3,7 +3,7 @@ import logging import re from functools import cached_property -from typing import Literal +from typing import ClassVar, Literal from pydantic import BaseModel from typing_extensions import override @@ -36,7 +36,7 @@ class Config(CamelCaseConfig): class KubernetesApp(PipelineComponent): """Base kubernetes app""" - type: Literal["kubernetes-app"] = "kubernetes-app" + type: ClassVar[Literal["kubernetes-app"]] = "kubernetes-app" app: KubernetesAppConfig version: str | None = None diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index ded790012..ee72d1801 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import ClassVar, Literal from pydantic import BaseConfig, Extra from typing_extensions import override @@ -20,7 +20,7 @@ class ProducerApp(KafkaApp): This producer holds configuration to use as values for the streams bootstrap produce helm chart. """ - type: Literal["producer"] = "producer" + type: ClassVar[Literal["producer"]] = "producer" app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 56a64353a..cd631161e 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal +from typing import ClassVar, Literal from pydantic import BaseConfig, Extra from typing_extensions import override @@ -16,7 +16,7 @@ class StreamsApp(KafkaApp): StreamsApp component that configures a streams bootstrap app """ - type: Literal["streams-app"] = "streams-app" + type: ClassVar[Literal["streams-app"]] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): diff --git a/tests/cli/test_registry.py b/tests/cli/test_registry.py index 334f16ac2..0b944404b 100644 --- a/tests/cli/test_registry.py +++ b/tests/cli/test_registry.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal +from typing import ClassVar, Literal import pytest from pydantic import BaseModel @@ -10,12 +10,12 @@ class SubComponent(PipelineComponent): - type: Literal["sub-component"] = "sub-component" + type: ClassVar[Literal["sub-component"]] = "sub-component" pass class SubSubComponent(SubComponent): - type: Literal["sub-sub-component"] = "sub-sub-component" + type: ClassVar[Literal["sub-sub-component"]] = "sub-sub-component" pass diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index 2c37f9b71..31b4d4a07 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Literal +from typing import ClassVar, Literal from unittest.mock import MagicMock import pytest @@ -17,7 +17,7 @@ class TestParentModel(BaseDefaultsComponent): __test__ = False - type: Literal["parent"] = "parent" + type: ClassVar[Literal["parent"]] = "parent" name: str | None = None value: float | None = None hard_coded: str = "hard_coded_value" @@ -25,14 +25,14 @@ class TestParentModel(BaseDefaultsComponent): class TestChildModel(TestParentModel): __test__ = False - type: Literal["child"] = "child" + type: ClassVar[Literal["child"]] = "child" nice: dict | None = None another_hard_coded: str = "another_hard_coded_value" class TestGrandChildModel(TestChildModel): __test__ = False - type: Literal["grand-child"] = "grand-child" + type: ClassVar[Literal["grand-child"]] = "grand-child" grand_child: str | None = None @@ -126,7 +126,7 @@ def test_env_var_substitution( self, config: PipelineConfig, handlers: ComponentHandlers ): class TestEnvVarModel(BaseDefaultsComponent): - type: Literal["env-var-test"] = "env-var-test" + type: ClassVar[Literal["env-var-test"]] = "env-var-test" name: str | None = None os.environ["pipeline_name"] = str(DEFAULTS_PATH) diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 59fabe370..10e204e42 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Literal +from typing import ClassVar, Literal from unittest.mock import MagicMock import pytest @@ -72,7 +72,7 @@ def streams_app( def test_set_topics(self, config: PipelineConfig, handlers: ComponentHandlers): class AnotherType(StreamsApp): - type: Literal["test"] = "test" + type: ClassVar[Literal["test"]] = "test" streams_app = AnotherType( config=config, diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index b140e0805..de492ec51 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Any, ClassVar, Literal from schema_registry.client.schema import AvroSchema from typing_extensions import override @@ -14,11 +14,11 @@ class ImportProducer(ProducerApp): - type: Literal["scheduled-producer"] = "scheduled-producer" + type: ClassVar[Literal["scheduled-producer"]] = "scheduled-producer" class Converter(StreamsApp): - type: Literal["converter"] = "converter" + type: ClassVar[Literal["converter"]] = "converter" class SubStreamsApp(StreamsApp): @@ -28,11 +28,11 @@ class SubStreamsApp(StreamsApp): class Filter(SubStreamsApp): """Subsubclass of StreamsApp to test inheritance.""" - type: Literal["filter"] = "filter" + type: ClassVar[Literal["filter"]] = "filter" class InflateStep(StreamsApp): - type: Literal["should-inflate"] = "should-inflate" + type: ClassVar[Literal["should-inflate"]] = "should-inflate" @override def inflate(self) -> list[PipelineComponent]: diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index d7e217c50..cf0e13e3a 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import ClassVar, Literal from typing_extensions import override @@ -10,15 +10,15 @@ class ImportProducer(ProducerApp): - type: Literal["scheduled-producer"] = "scheduled-producer" + type: ClassVar[Literal["scheduled-producer"]] = "scheduled-producer" class Converter(StreamsApp): - type: Literal["converter"] = "converter" + type: ClassVar[Literal["converter"]] = "converter" class InflateStep(StreamsApp): - type: Literal["should-inflate"] = "should-inflate" + type: ClassVar[Literal["should-inflate"]] = "should-inflate" @override def inflate(self) -> list[PipelineComponent]: From 89cfeb3562ae04619454a94c564d6b046507cd87 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 13:30:02 +0100 Subject: [PATCH 11/44] Workaround Pydantic discriminator --- .../base_defaults_component.py | 2 +- .../base_components/kafka_connect.py | 2 + .../producer/producer_app.py | 1 + .../streams_bootstrap/streams/streams_app.py | 1 + schema.json | 64 +++++++++---------- scratch_schema.py | 8 ++- 6 files changed, 43 insertions(+), 35 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index c47a36a52..2865b82e6 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -17,7 +17,7 @@ class BaseDefaultsComponent(BaseModel): - type: ClassVar[str] = Field(..., const=True) # component type discriminator + type: ClassVar[str] # component type enrich: bool = Field(default=False, exclude=True) config: PipelineConfig = Field(default=..., exclude=True) diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 8545d0c82..fa1e3d4bd 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -65,6 +65,7 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): type: ClassVar[Literal["kafka-source-connector"]] = "kafka-source-connector" + schema_type: Literal["kafka-source-connector"] = "kafka-source-connector" @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -91,6 +92,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): type: ClassVar[Literal["kafka-sink-connector"]] = "kafka-sink-connector" + schema_type: Literal["kafka-sink-connector"] = "kafka-sink-connector" @override def add_input_topics(self, topics: list[str]) -> None: diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index ee72d1801..a05be9b47 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -21,6 +21,7 @@ class ProducerApp(KafkaApp): """ type: ClassVar[Literal["producer"]] = "producer" + schema_type: Literal["producer"] = "producer" app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index cd631161e..62d09efdf 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -17,6 +17,7 @@ class StreamsApp(KafkaApp): """ type: ClassVar[Literal["streams-app"]] = "streams-app" + schema_type: Literal["streams-app"] = "streams-app" app: StreamsAppConfig class Config(BaseConfig): diff --git a/schema.json b/schema.json index d67ae1e34..8c6327a91 100644 --- a/schema.json +++ b/schema.json @@ -676,14 +676,6 @@ "description": "StreamsApp component that configures a streams bootstrap app", "type": "object", "properties": { - "type": { - "title": "Type", - "default": "streams-app", - "enum": [ - "streams-app" - ], - "type": "string" - }, "enrich": { "title": "Enrich", "default": false, @@ -708,6 +700,14 @@ "version": { "title": "Version", "type": "string" + }, + "type": { + "title": "Schema Type", + "default": "streams-app", + "enum": [ + "streams-app" + ], + "type": "string" } }, "required": [ @@ -771,14 +771,6 @@ "description": "Producer component\n\nThis producer holds configuration to use as values for the streams bootstrap produce helm chart.", "type": "object", "properties": { - "type": { - "title": "Type", - "default": "producer", - "enum": [ - "producer" - ], - "type": "string" - }, "enrich": { "title": "Enrich", "default": false, @@ -803,6 +795,14 @@ "version": { "title": "Version", "type": "string" + }, + "type": { + "title": "Schema Type", + "default": "producer", + "enum": [ + "producer" + ], + "type": "string" } }, "required": [ @@ -820,14 +820,6 @@ "title": "KafkaSourceConnector", "type": "object", "properties": { - "type": { - "title": "Type", - "default": "kafka-source-connector", - "enum": [ - "kafka-source-connector" - ], - "type": "string" - }, "enrich": { "title": "Enrich", "default": false, @@ -848,6 +840,14 @@ }, "to": { "$ref": "#/definitions/ToSection" + }, + "type": { + "title": "Schema Type", + "default": "kafka-source-connector", + "enum": [ + "kafka-source-connector" + ], + "type": "string" } }, "required": [ @@ -860,14 +860,6 @@ "title": "KafkaSinkConnector", "type": "object", "properties": { - "type": { - "title": "Type", - "default": "kafka-sink-connector", - "enum": [ - "kafka-sink-connector" - ], - "type": "string" - }, "enrich": { "title": "Enrich", "default": false, @@ -888,6 +880,14 @@ }, "to": { "$ref": "#/definitions/ToSection" + }, + "type": { + "title": "Schema Type", + "default": "kafka-sink-connector", + "enum": [ + "kafka-sink-connector" + ], + "type": "string" } }, "required": [ diff --git a/scratch_schema.py b/scratch_schema.py index 30c16a1c5..ff500e890 100644 --- a/scratch_schema.py +++ b/scratch_schema.py @@ -18,11 +18,15 @@ ) -AnnotatedPipelineComponent = Annotated[ComponentType, Field(discriminator="type")] +AnnotatedPipelineComponent = Annotated[ + ComponentType, Field(discriminator="schema_type") +] + schema = schema_json_of( Sequence[AnnotatedPipelineComponent], title="kpops pipeline schema", by_alias=True, indent=4, -) +).replace("schema_type", "type") + print(schema) From 09550befe420db56409f496ff6fdd6c7ed9aae35 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 13:34:34 +0100 Subject: [PATCH 12/44] Exclude config & enrich from schema --- .../base_defaults_component.py | 7 +- schema.json | 443 ------------------ 2 files changed, 3 insertions(+), 447 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 2865b82e6..fb766466b 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -7,7 +7,7 @@ from typing import ClassVar, TypeVar import typer -from pydantic import BaseConfig, BaseModel, Field, PrivateAttr +from pydantic import BaseConfig, BaseModel, PrivateAttr from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -19,15 +19,14 @@ class BaseDefaultsComponent(BaseModel): type: ClassVar[str] # component type - enrich: bool = Field(default=False, exclude=True) - config: PipelineConfig = Field(default=..., exclude=True) + _config: PipelineConfig = PrivateAttr() _handlers: ComponentHandlers = PrivateAttr() class Config(BaseConfig): arbitrary_types_allowed = True def __init__(self, **kwargs) -> None: - self._handlers = kwargs["handlers"] + self._handlers: ComponentHandlers = kwargs["handlers"] if kwargs.get("enrich", True): kwargs = self.extend_with_defaults(kwargs) super().__init__(**kwargs) diff --git a/schema.json b/schema.json index 8c6327a91..62730d536 100644 --- a/schema.json +++ b/schema.json @@ -27,413 +27,6 @@ ] }, "definitions": { - "TopicNameConfig": { - "title": "TopicNameConfig", - "description": "Base class for settings, allowing values to be overridden by environment variables.\n\nThis is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),\nHeroku and any 12 factor app design.", - "type": "object", - "properties": { - "default_output_topic_name": { - "title": "Default Output Topic Name", - "description": "Configures the value for the variable ${output_topic_name}", - "default": "${pipeline_name}-${component_name}", - "env_names": [ - "default_output_topic_name" - ], - "type": "string" - }, - "default_error_topic_name": { - "title": "Default Error Topic Name", - "description": "Configures the value for the variable ${error_topic_name}", - "default": "${pipeline_name}-${component_name}-error", - "env_names": [ - "default_error_topic_name" - ], - "type": "string" - } - }, - "additionalProperties": false - }, - "HelmConfig": { - "title": "HelmConfig", - "type": "object", - "properties": { - "context": { - "title": "Context", - "env_names": [ - "context" - ], - "type": "string" - }, - "debug": { - "title": "Debug", - "default": false, - "env_names": [ - "debug" - ], - "type": "boolean" - } - }, - "additionalProperties": false - }, - "HelmDiffConfig": { - "title": "HelmDiffConfig", - "type": "object", - "properties": { - "enable": { - "title": "Enable", - "default": false, - "env_names": [ - "enable" - ], - "type": "boolean" - }, - "ignore": { - "title": "Ignore", - "env_names": [ - "ignore" - ], - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true - } - }, - "additionalProperties": false - }, - "RepoAuthFlags": { - "title": "RepoAuthFlags", - "type": "object", - "properties": { - "username": { - "title": "Username", - "env_names": [ - "username" - ], - "type": "string" - }, - "password": { - "title": "Password", - "env_names": [ - "password" - ], - "type": "string" - }, - "ca_file": { - "title": "Ca File", - "env_names": [ - "ca_file" - ], - "type": "string", - "format": "path" - }, - "insecure_skip_tls_verify": { - "title": "Insecure Skip Tls Verify", - "default": false, - "env_names": [ - "insecure_skip_tls_verify" - ], - "type": "boolean" - } - }, - "additionalProperties": false - }, - "HelmRepoConfig": { - "title": "HelmRepoConfig", - "type": "object", - "properties": { - "repository_name": { - "title": "Repository Name", - "env_names": [ - "repository_name" - ], - "type": "string" - }, - "url": { - "title": "Url", - "env_names": [ - "url" - ], - "type": "string" - }, - "repo_auth_flags": { - "title": "Repo Auth Flags", - "env_names": [ - "repo_auth_flags" - ], - "allOf": [ - { - "$ref": "#/definitions/RepoAuthFlags" - } - ] - } - }, - "required": [ - "repository_name", - "url" - ], - "additionalProperties": false - }, - "KafkaConnectResetterHelmConfig": { - "title": "KafkaConnectResetterHelmConfig", - "description": "Base class for settings, allowing values to be overridden by environment variables.\n\nThis is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),\nHeroku and any 12 factor app design.", - "type": "object", - "properties": { - "helm_config": { - "title": "Helm Config", - "description": "Configuration of Kafka connect resetter Helm Chart", - "default": { - "repository_name": "bakdata-kafka-connect-resetter", - "url": "https://bakdata.github.io/kafka-connect-resetter/", - "repo_auth_flags": { - "username": null, - "password": null, - "ca_file": null, - "insecure_skip_tls_verify": false - } - }, - "env_names": [ - "helm_config" - ], - "allOf": [ - { - "$ref": "#/definitions/HelmRepoConfig" - } - ] - }, - "version": { - "title": "Version", - "default": "1.0.4", - "env_names": [ - "version" - ], - "type": "string" - }, - "helm_values": { - "title": "Helm Values", - "description": "Overriding Kafka Connect Resetter Helm values. E.g. to override the Image Tag etc.", - "default": {}, - "env_names": [ - "helm_values" - ], - "type": "object" - }, - "namespace": { - "title": "Namespace", - "default": "", - "env_names": [ - "namespace" - ], - "type": "string" - } - }, - "additionalProperties": false - }, - "PipelineConfig": { - "title": "PipelineConfig", - "description": "Base class for settings, allowing values to be overridden by environment variables.\n\nThis is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),\nHeroku and any 12 factor app design.", - "type": "object", - "properties": { - "defaults_path": { - "title": "Defaults Path", - "description": "The path to the folder containing the defaults file and the environment defaults files.", - "env_names": [ - "defaults_path" - ], - "type": "string", - "format": "path" - }, - "environment": { - "title": "Environment", - "description": "The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).", - "env": "KPOPS_ENVIRONMENT", - "example": "development", - "env_names": [ - "kpops_environment" - ], - "type": "string" - }, - "broker": { - "title": "Broker", - "description": "The kafka broker address.", - "env": "KPOPS_KAFKA_BROKER", - "env_names": [ - "kpops_kafka_broker" - ], - "type": "string" - }, - "defaults_filename_prefix": { - "title": "Defaults Filename Prefix", - "description": "The name of the defaults file and the prefix of the defaults environment file.", - "default": "defaults", - "env_names": [ - "defaults_filename_prefix" - ], - "type": "string" - }, - "topic_name_config": { - "title": "Topic Name Config", - "description": "Configure the topic name variables you can use in the pipeline definition.", - "default": { - "default_output_topic_name": "${pipeline_name}-${component_name}", - "default_error_topic_name": "${pipeline_name}-${component_name}-error" - }, - "env_names": [ - "topic_name_config" - ], - "allOf": [ - { - "$ref": "#/definitions/TopicNameConfig" - } - ] - }, - "kafka_rest_host": { - "title": "Kafka Rest Host", - "description": "Address to the rest proxy REST API.", - "env": "KPOPS_REST_PROXY_HOST", - "example": "http://localhost:8082", - "env_names": [ - "kpops_rest_proxy_host" - ], - "type": "string" - }, - "kafka_connect_host": { - "title": "Kafka Connect Host", - "description": "Address to the kafka connect REST API.", - "env": "KPOPS_CONNECT_HOST", - "example": "http://localhost:8083", - "env_names": [ - "kpops_connect_host" - ], - "type": "string" - }, - "timeout": { - "title": "Timeout", - "description": "The timeout in seconds that specifies when actions like deletion or deploy timeout.", - "default": 300, - "env": "KPOPS_TIMEOUT", - "env_names": [ - "kpops_timeout" - ], - "type": "integer" - }, - "pipeline_prefix": { - "title": "Pipeline Prefix", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "default": "${pipeline_name}-", - "env": "KPOPS_PIPELINE_PREFIX", - "env_names": [ - "kpops_pipeline_prefix" - ], - "type": "string" - }, - "helm_config": { - "title": "Helm Config", - "default": { - "context": null, - "debug": false - }, - "env_names": [ - "helm_config" - ], - "allOf": [ - { - "$ref": "#/definitions/HelmConfig" - } - ] - }, - "helm_diff_config": { - "title": "Helm Diff Config", - "default": { - "enable": false, - "ignore": [] - }, - "env_names": [ - "helm_diff_config" - ], - "allOf": [ - { - "$ref": "#/definitions/HelmDiffConfig" - } - ] - }, - "streams_bootstrap_helm_config": { - "title": "Streams Bootstrap Helm Config", - "description": "Configuration for Streams Bootstrap Helm Charts", - "default": { - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/", - "repo_auth_flags": { - "username": null, - "password": null, - "ca_file": null, - "insecure_skip_tls_verify": false - } - }, - "env_names": [ - "streams_bootstrap_helm_config" - ], - "allOf": [ - { - "$ref": "#/definitions/HelmRepoConfig" - } - ] - }, - "kafka_connect_resetter_config": { - "title": "Kafka Connect Resetter Config", - "description": "Configuration of kafka connect resetter helm chart and values. This is used for cleaning/resettting Kafka connectors, see https://github.com/bakdata/kafka-connect-resetter", - "default": { - "helm_config": { - "repository_name": "bakdata-kafka-connect-resetter", - "url": "https://bakdata.github.io/kafka-connect-resetter/", - "repo_auth_flags": { - "username": null, - "password": null, - "ca_file": null, - "insecure_skip_tls_verify": false - } - }, - "version": "1.0.4", - "helm_values": {}, - "namespace": "" - }, - "env_names": [ - "kafka_connect_resetter_config" - ], - "allOf": [ - { - "$ref": "#/definitions/KafkaConnectResetterHelmConfig" - } - ] - }, - "retain_clean_jobs": { - "title": "Retain Clean Jobs", - "description": "Whether to retain clean up jobs in the cluster or uninstall the, after completion.", - "default": false, - "env": "KPOPS_RETAIN_CLEAN_JOBS", - "env_names": [ - "kpops_retain_clean_jobs" - ], - "type": "boolean" - }, - "schema_registry_url": { - "title": "Schema Registry Url", - "description": "The URL to schema registry.", - "example": "http://localhost:8081", - "env": "KPOPS_SCHEMA_REGISTRY_URL", - "env_names": [ - "kpops_schema_registry_url" - ], - "type": "string" - } - }, - "required": [ - "defaults_path", - "environment", - "broker" - ], - "additionalProperties": false - }, "InputTopicTypes": { "title": "InputTopicTypes", "description": "An enumeration.", @@ -676,14 +269,6 @@ "description": "StreamsApp component that configures a streams bootstrap app", "type": "object", "properties": { - "enrich": { - "title": "Enrich", - "default": false, - "type": "boolean" - }, - "config": { - "$ref": "#/definitions/PipelineConfig" - }, "name": { "title": "Name", "type": "string" @@ -711,7 +296,6 @@ } }, "required": [ - "config", "name", "app" ] @@ -771,14 +355,6 @@ "description": "Producer component\n\nThis producer holds configuration to use as values for the streams bootstrap produce helm chart.", "type": "object", "properties": { - "enrich": { - "title": "Enrich", - "default": false, - "type": "boolean" - }, - "config": { - "$ref": "#/definitions/PipelineConfig" - }, "name": { "title": "Name", "type": "string" @@ -806,7 +382,6 @@ } }, "required": [ - "config", "name", "app" ] @@ -820,14 +395,6 @@ "title": "KafkaSourceConnector", "type": "object", "properties": { - "enrich": { - "title": "Enrich", - "default": false, - "type": "boolean" - }, - "config": { - "$ref": "#/definitions/PipelineConfig" - }, "name": { "title": "Name", "type": "string" @@ -851,7 +418,6 @@ } }, "required": [ - "config", "name", "app" ] @@ -860,14 +426,6 @@ "title": "KafkaSinkConnector", "type": "object", "properties": { - "enrich": { - "title": "Enrich", - "default": false, - "type": "boolean" - }, - "config": { - "$ref": "#/definitions/PipelineConfig" - }, "name": { "title": "Name", "type": "string" @@ -891,7 +449,6 @@ } }, "required": [ - "config", "name", "app" ] From 7c9aeb7eb2f5b7df0004f17694eb351672d2e1cc Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 13:44:52 +0100 Subject: [PATCH 13/44] Fix issues --- .../base_components/base_defaults_component.py | 4 ++-- kpops/components/base_components/kafka_app.py | 4 ++-- kpops/components/base_components/kafka_connect.py | 6 +++--- kpops/components/base_components/kubernetes_app.py | 4 ++-- .../streams_bootstrap/producer/producer_app.py | 2 +- .../streams_bootstrap/streams/streams_app.py | 2 +- tests/cli/test_registry.py | 6 +++--- tests/components/test_base_defaults_component.py | 10 +++++----- tests/components/test_streams_app.py | 4 ++-- tests/pipeline/test_components/components.py | 10 +++++----- .../components.py | 8 ++++---- 11 files changed, 30 insertions(+), 30 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index fb766466b..382bdfe51 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -7,7 +7,7 @@ from typing import ClassVar, TypeVar import typer -from pydantic import BaseConfig, BaseModel, PrivateAttr +from pydantic import BaseConfig, BaseModel, Field, PrivateAttr from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -17,7 +17,7 @@ class BaseDefaultsComponent(BaseModel): - type: ClassVar[str] # component type + type: ClassVar[str] = Field(..., const=True) _config: PipelineConfig = PrivateAttr() _handlers: ComponentHandlers = PrivateAttr() diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 341cd0622..96e41b2fa 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import ClassVar, Literal +from typing import ClassVar from pydantic import BaseModel, Extra from typing_extensions import override @@ -38,7 +38,7 @@ class KafkaApp(KubernetesApp): Producer or streaming apps should inherit from this class. """ - type: ClassVar[Literal["kafka-app"]] = "kafka-app" + type: ClassVar[str] = "kafka-app" app: KafkaAppConfig def __init__(self, **kwargs) -> None: diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index fa1e3d4bd..9f29e76ea 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -14,7 +14,7 @@ class KafkaConnector(PipelineComponent, ABC): - type: ClassVar[Literal["kafka-connect"]] = "kafka-connect" + type: ClassVar[str] = "kafka-connect" app: KafkaConnectConfig def __init__(self, **kwargs) -> None: @@ -64,7 +64,7 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): - type: ClassVar[Literal["kafka-source-connector"]] = "kafka-source-connector" + type: ClassVar[str] = "kafka-source-connector" schema_type: Literal["kafka-source-connector"] = "kafka-source-connector" @override @@ -91,7 +91,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): - type: ClassVar[Literal["kafka-sink-connector"]] = "kafka-sink-connector" + type: ClassVar[str] = "kafka-sink-connector" schema_type: Literal["kafka-sink-connector"] = "kafka-sink-connector" @override diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 112d2edb9..857e13096 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -3,7 +3,7 @@ import logging import re from functools import cached_property -from typing import ClassVar, Literal +from typing import ClassVar from pydantic import BaseModel from typing_extensions import override @@ -36,7 +36,7 @@ class Config(CamelCaseConfig): class KubernetesApp(PipelineComponent): """Base kubernetes app""" - type: ClassVar[Literal["kubernetes-app"]] = "kubernetes-app" + type: ClassVar[str] = "kubernetes-app" app: KubernetesAppConfig version: str | None = None diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index a05be9b47..4a6654ca8 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -20,7 +20,7 @@ class ProducerApp(KafkaApp): This producer holds configuration to use as values for the streams bootstrap produce helm chart. """ - type: ClassVar[Literal["producer"]] = "producer" + type: ClassVar[str] = "producer" schema_type: Literal["producer"] = "producer" app: ProducerValues diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 62d09efdf..26d732a85 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -16,7 +16,7 @@ class StreamsApp(KafkaApp): StreamsApp component that configures a streams bootstrap app """ - type: ClassVar[Literal["streams-app"]] = "streams-app" + type: ClassVar[str] = "streams-app" schema_type: Literal["streams-app"] = "streams-app" app: StreamsAppConfig diff --git a/tests/cli/test_registry.py b/tests/cli/test_registry.py index 0b944404b..019c55ae3 100644 --- a/tests/cli/test_registry.py +++ b/tests/cli/test_registry.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import ClassVar, Literal +from typing import ClassVar import pytest from pydantic import BaseModel @@ -10,12 +10,12 @@ class SubComponent(PipelineComponent): - type: ClassVar[Literal["sub-component"]] = "sub-component" + type: ClassVar[str] = "sub-component" pass class SubSubComponent(SubComponent): - type: ClassVar[Literal["sub-sub-component"]] = "sub-sub-component" + type: ClassVar[str] = "sub-sub-component" pass diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index 31b4d4a07..f0ab66e7c 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import ClassVar, Literal +from typing import ClassVar from unittest.mock import MagicMock import pytest @@ -17,7 +17,7 @@ class TestParentModel(BaseDefaultsComponent): __test__ = False - type: ClassVar[Literal["parent"]] = "parent" + type: ClassVar[str] = "parent" name: str | None = None value: float | None = None hard_coded: str = "hard_coded_value" @@ -25,14 +25,14 @@ class TestParentModel(BaseDefaultsComponent): class TestChildModel(TestParentModel): __test__ = False - type: ClassVar[Literal["child"]] = "child" + type: ClassVar[str] = "child" nice: dict | None = None another_hard_coded: str = "another_hard_coded_value" class TestGrandChildModel(TestChildModel): __test__ = False - type: ClassVar[Literal["grand-child"]] = "grand-child" + type: ClassVar[str] = "grand-child" grand_child: str | None = None @@ -126,7 +126,7 @@ def test_env_var_substitution( self, config: PipelineConfig, handlers: ComponentHandlers ): class TestEnvVarModel(BaseDefaultsComponent): - type: ClassVar[Literal["env-var-test"]] = "env-var-test" + type: ClassVar[str] = "env-var-test" name: str | None = None os.environ["pipeline_name"] = str(DEFAULTS_PATH) diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 10e204e42..00c5a7f86 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import ClassVar, Literal +from typing import ClassVar from unittest.mock import MagicMock import pytest @@ -72,7 +72,7 @@ def streams_app( def test_set_topics(self, config: PipelineConfig, handlers: ComponentHandlers): class AnotherType(StreamsApp): - type: ClassVar[Literal["test"]] = "test" + type: ClassVar[str] = "test" streams_app = AnotherType( config=config, diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index de492ec51..b7f7975b9 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -1,4 +1,4 @@ -from typing import Any, ClassVar, Literal +from typing import Any, ClassVar from schema_registry.client.schema import AvroSchema from typing_extensions import override @@ -14,11 +14,11 @@ class ImportProducer(ProducerApp): - type: ClassVar[Literal["scheduled-producer"]] = "scheduled-producer" + type: ClassVar[str] = "scheduled-producer" class Converter(StreamsApp): - type: ClassVar[Literal["converter"]] = "converter" + type: ClassVar[str] = "converter" class SubStreamsApp(StreamsApp): @@ -28,11 +28,11 @@ class SubStreamsApp(StreamsApp): class Filter(SubStreamsApp): """Subsubclass of StreamsApp to test inheritance.""" - type: ClassVar[Literal["filter"]] = "filter" + type: ClassVar[str] = "filter" class InflateStep(StreamsApp): - type: ClassVar[Literal["should-inflate"]] = "should-inflate" + type: ClassVar[str] = "should-inflate" @override def inflate(self) -> list[PipelineComponent]: diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index cf0e13e3a..48c051cff 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -1,4 +1,4 @@ -from typing import ClassVar, Literal +from typing import ClassVar from typing_extensions import override @@ -10,15 +10,15 @@ class ImportProducer(ProducerApp): - type: ClassVar[Literal["scheduled-producer"]] = "scheduled-producer" + type: ClassVar[str] = "scheduled-producer" class Converter(StreamsApp): - type: ClassVar[Literal["converter"]] = "converter" + type: ClassVar[str] = "converter" class InflateStep(StreamsApp): - type: ClassVar[Literal["should-inflate"]] = "should-inflate" + type: ClassVar[str] = "should-inflate" @override def inflate(self) -> list[PipelineComponent]: From 370a28d78b040c9714cb3717c7bf9b3665c158d5 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 13:46:19 +0100 Subject: [PATCH 14/44] Add property --- kpops/components/base_components/base_defaults_component.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 382bdfe51..6d4279d86 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -31,6 +31,10 @@ def __init__(self, **kwargs) -> None: kwargs = self.extend_with_defaults(kwargs) super().__init__(**kwargs) + @property + def config(self) -> PipelineConfig: + return self._config + @property def handlers(self) -> ComponentHandlers: return self._handlers From 9c7b8eb1f2cc8f8e81cf5cc27882f0e8634a7a6c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 13:59:18 +0100 Subject: [PATCH 15/44] Update config & handlers attrs --- .../base_defaults_component.py | 4 -- kpops/components/base_components/kafka_app.py | 4 +- .../base_components/kafka_connect.py | 4 +- .../base_components/kubernetes_app.py | 4 +- .../base_components/pipeline_component.py | 4 +- .../producer/producer_app.py | 8 ++-- .../streams_bootstrap/streams/streams_app.py | 8 ++-- .../test_base_defaults_component.py | 11 ++--- tests/components/test_kafka_connect.py | 48 +++++++------------ tests/components/test_producer_app.py | 8 ++-- tests/components/test_streams_app.py | 2 +- tests/pipeline/test_components/components.py | 15 ++++-- .../components.py | 6 ++- 13 files changed, 55 insertions(+), 71 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 6d4279d86..382bdfe51 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -31,10 +31,6 @@ def __init__(self, **kwargs) -> None: kwargs = self.extend_with_defaults(kwargs) super().__init__(**kwargs) - @property - def config(self) -> PipelineConfig: - return self._config - @property def handlers(self) -> ComponentHandlers: return self._handlers diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 96e41b2fa..3b646ac2d 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -44,12 +44,12 @@ class KafkaApp(KubernetesApp): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self.app.streams.brokers = substitute( - self.app.streams.brokers, {"broker": self.config.broker} + self.app.streams.brokers, {"broker": self._config.broker} ) if self.app.streams.schema_registry_url: self.app.streams.schema_registry_url = substitute( self.app.streams.schema_registry_url, - {"schema_registry_url": self.config.schema_registry_url}, + {"schema_registry_url": self._config.schema_registry_url}, ) @property diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 9f29e76ea..0c71b69b7 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -85,7 +85,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: connector_name=self.name, connector_type=KafkaConnectorType.SOURCE, dry_run=dry_run, - retain_clean_jobs=self.config.retain_clean_jobs, + retain_clean_jobs=self._config.retain_clean_jobs, offset_topic=os.environ[f"{ENV_PREFIX}KAFKA_CONNECT_RESETTER_OFFSET_TOPIC"], ) @@ -125,6 +125,6 @@ def __clean_sink_connector( connector_name=self.name, connector_type=KafkaConnectorType.SINK, dry_run=dry_run, - retain_clean_jobs=self.config.retain_clean_jobs, + retain_clean_jobs=self._config.retain_clean_jobs, delete_consumer_group=delete_consumer_group, ) diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 857e13096..6b99281ff 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -50,7 +50,7 @@ def __init__(self, **kwargs): @cached_property def helm(self) -> Helm: - helm = Helm(self.config.helm_config) + helm = Helm(self._config.helm_config) if self.helm_repo_config is not None: helm.add_repo( self.helm_repo_config.repository_name, @@ -61,7 +61,7 @@ def helm(self) -> Helm: @cached_property def helm_diff(self) -> HelmDiff: - return HelmDiff(self.config.helm_diff_config) + return HelmDiff(self._config.helm_diff_config) @property def helm_release_name(self) -> str: diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 9ac1367a2..8ac02d24b 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -59,12 +59,12 @@ def substitute_component_variables(self, topic_name: str) -> str: :return: final topic name """ error_topic_name = self.substitute_component_names( - self.config.topic_name_config.default_error_topic_name, + self._config.topic_name_config.default_error_topic_name, self.type, **os.environ, ) output_topic_name = self.substitute_component_names( - self.config.topic_name_config.default_output_topic_name, + self._config.topic_name_config.default_output_topic_name, self.type, **os.environ, ) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 4a6654ca8..a77ee22e3 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -45,22 +45,22 @@ def add_extra_output_topic(self, topic_name: str, role: str) -> None: @override def get_helm_chart(self) -> str: - return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.PRODUCER_APP.value}" + return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.PRODUCER_APP.value}" @property @override def clean_up_helm_chart(self) -> str: - return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" + return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" @property @override def helm_repo_config(self) -> HelmRepoConfig | None: - return self.config.streams_bootstrap_helm_config + return self._config.streams_bootstrap_helm_config @override def clean(self, dry_run: bool) -> None: self._run_clean_up_job( values=self.to_helm_values(), dry_run=dry_run, - retain_clean_jobs=self.config.retain_clean_jobs, + retain_clean_jobs=self._config.retain_clean_jobs, ) diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 26d732a85..3ffd2049d 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -57,17 +57,17 @@ def add_extra_output_topic(self, topic_name: str, role: str) -> None: @override def get_helm_chart(self) -> str: - return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.STREAMS_APP.value}" + return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.STREAMS_APP.value}" @property @override def helm_repo_config(self) -> HelmRepoConfig | None: - return self.config.streams_bootstrap_helm_config + return self._config.streams_bootstrap_helm_config @property @override def clean_up_helm_chart(self) -> str: - return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" + return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" @override def reset(self, dry_run: bool) -> None: @@ -83,7 +83,7 @@ def __run_streams_clean_up_job(self, dry_run: bool, delete_output: bool) -> None self._run_clean_up_job( values=values, dry_run=dry_run, - retain_clean_jobs=self.config.retain_clean_jobs, + retain_clean_jobs=self._config.retain_clean_jobs, ) def __substitute_autoscaling_topic_names(self) -> None: diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index f0ab66e7c..400cb062c 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -57,7 +57,7 @@ class TestBaseDefaultsComponent: def test_inherit_defaults( self, config: PipelineConfig, handlers: ComponentHandlers ): - component = TestChildModel(config=config, **{"handlers": handlers}) + component = TestChildModel(**{"config": config, "handlers": handlers}) assert ( component.name == "fake-child-name" @@ -77,9 +77,9 @@ def test_inherit_defaults( def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): component = TestChildModel( - config=config, **{ "name": "name-defined-in-pipeline_generator", + "config": config, "handlers": handlers, }, ) @@ -103,7 +103,7 @@ def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): def test_multiple_generations( self, config: PipelineConfig, handlers: ComponentHandlers ): - component = TestGrandChildModel(config=config, **{"handlers": handlers}) + component = TestGrandChildModel(**{"config": config, "handlers": handlers}) assert ( component.name == "fake-child-name" @@ -130,10 +130,7 @@ class TestEnvVarModel(BaseDefaultsComponent): name: str | None = None os.environ["pipeline_name"] = str(DEFAULTS_PATH) - component = TestEnvVarModel( - config=config, - **{"handlers": handlers}, - ) + component = TestEnvVarModel(**{"config": config, "handlers": handlers}) assert component.name == str( DEFAULTS_PATH diff --git a/tests/components/test_kafka_connect.py b/tests/components/test_kafka_connect.py index c7e6916cc..bbbf45746 100644 --- a/tests/components/test_kafka_connect.py +++ b/tests/components/test_kafka_connect.py @@ -56,31 +56,28 @@ def test_connector_config_parsing( topic_name = "connector-topic" connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(**{"topics": topic_name}), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) assert getattr(connector.app, "topics") == topic_name topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(**{"topics.regex": topic_pattern}), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) assert getattr(connector.app, "topics.regex") == topic_pattern connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ "${error_topic_name}": TopicConfig(type=OutputTopicTypes.ERROR), } ), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) assert ( getattr(connector.app, "errors.deadletterqueue.topic.name") @@ -94,7 +91,6 @@ def test_from_section_parsing_input_topic( topic2 = "connector-topic2" connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(), from_=FromSection( topics={ @@ -102,7 +98,7 @@ def test_from_section_parsing_input_topic( topic2: FromTopic(type=InputTopicTypes.INPUT), } ), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) assert getattr(connector.app, "topics") == f"{topic1},{topic2}" @@ -116,12 +112,11 @@ def test_from_section_parsing_input_pattern( topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(), from_=FromSection( topics={topic_pattern: FromTopic(type=InputTopicTypes.INPUT_PATTERN)} ), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) assert getattr(connector.app, "topics.regex") == topic_pattern @@ -133,7 +128,6 @@ def test_deploy_order( ): connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -142,7 +136,7 @@ def test_deploy_order( ), } ), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) mock_create_topics = mocker.patch.object( @@ -175,7 +169,6 @@ def test_destroy( ): connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -184,7 +177,7 @@ def test_destroy( ), } ), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) mock_destroy_connector = mocker.patch.object( @@ -206,7 +199,6 @@ def test_clean( ): connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -215,7 +207,7 @@ def test_clean( ), } ), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) mock_delete_topics = mocker.patch.object( @@ -251,7 +243,6 @@ def test_reset( ): connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -260,7 +251,7 @@ def test_reset( ), } ), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) mock_delete_topics = mocker.patch.object( @@ -289,9 +280,8 @@ def test_clean_without_to( ): connector = KafkaSinkConnector( name="test-connector", - config=config, app=KafkaConnectConfig(), - **{"handlers": handlers}, + **{"config": config, "handlers": handlers}, ) mock_delete_topics = mocker.patch.object( @@ -320,14 +310,13 @@ def test_from_section_raises_exception( with pytest.raises(NotImplementedError): KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, app=KafkaConnectConfig(), from_=FromSection( topics={ "connector-topic": FromTopic(type=InputTopicTypes.INPUT), } ), + **{"config": config, "handlers": handlers}, ) def test_deploy_order( @@ -338,8 +327,6 @@ def test_deploy_order( ): connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -348,6 +335,7 @@ def test_deploy_order( ), } ), + **{"config": config, "handlers": handlers}, ) mock_create_topics = mocker.patch.object( @@ -384,8 +372,6 @@ def test_destroy( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -394,6 +380,7 @@ def test_destroy( ), } ), + **{"config": config, "handlers": handlers}, ) assert connector.handlers.connector_handler @@ -419,8 +406,6 @@ def test_clean( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -429,6 +414,7 @@ def test_clean( ), } ), + **{"config": config, "handlers": handlers}, ) assert connector.handlers.connector_handler @@ -469,9 +455,8 @@ def test_clean_without_to( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, app=KafkaConnectConfig(), + **{"config": config, "handlers": handlers}, ) assert connector.to is None @@ -506,8 +491,6 @@ def test_reset( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, - config=config, app=KafkaConnectConfig(), to=ToSection( topics={ @@ -516,6 +499,7 @@ def test_reset( ), } ), + **{"config": config, "handlers": handlers}, ) assert connector.handlers.connector_handler diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index f79cbc4a0..f1939577a 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -48,11 +48,11 @@ def producer_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> ProducerApp: return ProducerApp( - handlers=handlers, - config=config, **{ "type": "producer-app", "name": self.PRODUCER_APP_NAME, + "config": config, + "handlers": handlers, "version": "2.4.2", "app": { "namespace": "test-namespace", @@ -71,11 +71,11 @@ def producer_app( def test_output_topics(self, config: PipelineConfig, handlers: ComponentHandlers): producer_app = ProducerApp( - handlers=handlers, - config=config, **{ "type": "producer-app", "name": self.PRODUCER_APP_NAME, + "config": config, + "handlers": handlers, "app": { "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 00c5a7f86..a1d21dce0 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -75,10 +75,10 @@ class AnotherType(StreamsApp): type: ClassVar[str] = "test" streams_app = AnotherType( - config=config, **{ "type": "test", "name": self.STREAMS_APP_NAME, + "config": config, "handlers": handlers, "app": { "namespace": "test-namespace", diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index b7f7975b9..3efa0a6f3 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -3,6 +3,7 @@ from schema_registry.client.schema import AvroSchema from typing_extensions import override +from kpops.component_handlers.kafka_connect.model import KafkaConnectConfig from kpops.component_handlers.schema_handler.schema_provider import ( Schema, SchemaProvider, @@ -42,11 +43,15 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - handlers=self.handlers, - config=self.config, - app={ - "topics": topic_name, - "transforms.changeTopic.replacement": f"{topic_name}-index-v1", + app=KafkaConnectConfig( + **{ + "topics": topic_name, + "transforms.changeTopic.replacement": f"{topic_name}-index-v1", + } + ), + **{ + "config": self._config, + "handlers": self._handlers, }, ) inflate_steps.append(kafka_connector) diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 48c051cff..183367a63 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -28,14 +28,16 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - handlers=self.handlers, - config=self.config, app=KafkaConnectConfig( **{ "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", } ), + **{ + "config": self._config, + "handlers": self._handlers, + }, ) inflate_steps.append(kafka_connector) From 54be760a33035cefd10f8f15f0b7d26850d8f1c1 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 14:05:52 +0100 Subject: [PATCH 16/44] Rename schema codegen --- scratch_schema.py => gen_schema.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scratch_schema.py => gen_schema.py (100%) diff --git a/scratch_schema.py b/gen_schema.py similarity index 100% rename from scratch_schema.py rename to gen_schema.py From 1d5b5caa2bfabce02d2127f4b1cbf73552a6773f Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 14:13:34 +0100 Subject: [PATCH 17/44] Work around Pydantic quirks --- gen_schema.py | 19 +++++++++++++++++-- .../base_defaults_component.py | 11 +++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/gen_schema.py b/gen_schema.py index ff500e890..2dc67269e 100644 --- a/gen_schema.py +++ b/gen_schema.py @@ -1,6 +1,8 @@ -from typing import Annotated, Sequence +from typing import Annotated, Any, Sequence -from pydantic import Field, schema_json_of +from pydantic import Field, schema, schema_json_of +from pydantic.fields import ModelField +from pydantic.schema import SkipField from kpops.components.base_components.kafka_connect import ( KafkaSinkConnector, @@ -9,6 +11,19 @@ from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp +original_field_schema = schema.field_schema + + +# adapted from https://github.com/tiangolo/fastapi/issues/1378#issuecomment-764966955 +def field_schema(field: ModelField, **kwargs: Any) -> Any: + if field.field_info.exclude: + raise SkipField(f"{field.name} field is being hidden") + else: + return original_field_schema(field, **kwargs) + + +schema.field_schema = field_schema + ComponentType = ( StreamsApp | ProducerApp diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 382bdfe51..a758068e3 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -7,7 +7,7 @@ from typing import ClassVar, TypeVar import typer -from pydantic import BaseConfig, BaseModel, Field, PrivateAttr +from pydantic import BaseConfig, BaseModel, Field from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -19,22 +19,17 @@ class BaseDefaultsComponent(BaseModel): type: ClassVar[str] = Field(..., const=True) - _config: PipelineConfig = PrivateAttr() - _handlers: ComponentHandlers = PrivateAttr() + config: PipelineConfig = Field(..., exclude=True) + handlers: ComponentHandlers = Field(..., exclude=True) class Config(BaseConfig): arbitrary_types_allowed = True def __init__(self, **kwargs) -> None: - self._handlers: ComponentHandlers = kwargs["handlers"] if kwargs.get("enrich", True): kwargs = self.extend_with_defaults(kwargs) super().__init__(**kwargs) - @property - def handlers(self) -> ComponentHandlers: - return self._handlers - def extend_with_defaults(self, kwargs) -> dict: """ Merges tmp_defaults with all tmp_defaults for parent classes From 26281f23a956c78420d6a07534ba304ebb2791df Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 14:13:34 +0100 Subject: [PATCH 18/44] Work around Pydantic quirks --- .../components/base_components/base_defaults_component.py | 1 + kpops/components/base_components/kafka_app.py | 4 ++-- kpops/components/base_components/kafka_connect.py | 4 ++-- kpops/components/base_components/kubernetes_app.py | 4 ++-- kpops/components/base_components/pipeline_component.py | 4 ++-- .../components/streams_bootstrap/producer/producer_app.py | 8 ++++---- kpops/components/streams_bootstrap/streams/streams_app.py | 8 ++++---- tests/pipeline/test_components/components.py | 4 ++-- .../test_components_without_schema_handler/components.py | 4 ++-- 9 files changed, 21 insertions(+), 20 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index a758068e3..c537a2958 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -19,6 +19,7 @@ class BaseDefaultsComponent(BaseModel): type: ClassVar[str] = Field(..., const=True) + enrich: bool = Field(default=False, exclude=True) config: PipelineConfig = Field(..., exclude=True) handlers: ComponentHandlers = Field(..., exclude=True) diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 3b646ac2d..96e41b2fa 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -44,12 +44,12 @@ class KafkaApp(KubernetesApp): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self.app.streams.brokers = substitute( - self.app.streams.brokers, {"broker": self._config.broker} + self.app.streams.brokers, {"broker": self.config.broker} ) if self.app.streams.schema_registry_url: self.app.streams.schema_registry_url = substitute( self.app.streams.schema_registry_url, - {"schema_registry_url": self._config.schema_registry_url}, + {"schema_registry_url": self.config.schema_registry_url}, ) @property diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 0c71b69b7..9f29e76ea 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -85,7 +85,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: connector_name=self.name, connector_type=KafkaConnectorType.SOURCE, dry_run=dry_run, - retain_clean_jobs=self._config.retain_clean_jobs, + retain_clean_jobs=self.config.retain_clean_jobs, offset_topic=os.environ[f"{ENV_PREFIX}KAFKA_CONNECT_RESETTER_OFFSET_TOPIC"], ) @@ -125,6 +125,6 @@ def __clean_sink_connector( connector_name=self.name, connector_type=KafkaConnectorType.SINK, dry_run=dry_run, - retain_clean_jobs=self._config.retain_clean_jobs, + retain_clean_jobs=self.config.retain_clean_jobs, delete_consumer_group=delete_consumer_group, ) diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 6b99281ff..857e13096 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -50,7 +50,7 @@ def __init__(self, **kwargs): @cached_property def helm(self) -> Helm: - helm = Helm(self._config.helm_config) + helm = Helm(self.config.helm_config) if self.helm_repo_config is not None: helm.add_repo( self.helm_repo_config.repository_name, @@ -61,7 +61,7 @@ def helm(self) -> Helm: @cached_property def helm_diff(self) -> HelmDiff: - return HelmDiff(self._config.helm_diff_config) + return HelmDiff(self.config.helm_diff_config) @property def helm_release_name(self) -> str: diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 8ac02d24b..9ac1367a2 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -59,12 +59,12 @@ def substitute_component_variables(self, topic_name: str) -> str: :return: final topic name """ error_topic_name = self.substitute_component_names( - self._config.topic_name_config.default_error_topic_name, + self.config.topic_name_config.default_error_topic_name, self.type, **os.environ, ) output_topic_name = self.substitute_component_names( - self._config.topic_name_config.default_output_topic_name, + self.config.topic_name_config.default_output_topic_name, self.type, **os.environ, ) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index a77ee22e3..4a6654ca8 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -45,22 +45,22 @@ def add_extra_output_topic(self, topic_name: str, role: str) -> None: @override def get_helm_chart(self) -> str: - return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.PRODUCER_APP.value}" + return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.PRODUCER_APP.value}" @property @override def clean_up_helm_chart(self) -> str: - return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" + return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" @property @override def helm_repo_config(self) -> HelmRepoConfig | None: - return self._config.streams_bootstrap_helm_config + return self.config.streams_bootstrap_helm_config @override def clean(self, dry_run: bool) -> None: self._run_clean_up_job( values=self.to_helm_values(), dry_run=dry_run, - retain_clean_jobs=self._config.retain_clean_jobs, + retain_clean_jobs=self.config.retain_clean_jobs, ) diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 3ffd2049d..26d732a85 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -57,17 +57,17 @@ def add_extra_output_topic(self, topic_name: str, role: str) -> None: @override def get_helm_chart(self) -> str: - return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.STREAMS_APP.value}" + return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.STREAMS_APP.value}" @property @override def helm_repo_config(self) -> HelmRepoConfig | None: - return self._config.streams_bootstrap_helm_config + return self.config.streams_bootstrap_helm_config @property @override def clean_up_helm_chart(self) -> str: - return f"{self._config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" + return f"{self.config.streams_bootstrap_helm_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" @override def reset(self, dry_run: bool) -> None: @@ -83,7 +83,7 @@ def __run_streams_clean_up_job(self, dry_run: bool, delete_output: bool) -> None self._run_clean_up_job( values=values, dry_run=dry_run, - retain_clean_jobs=self._config.retain_clean_jobs, + retain_clean_jobs=self.config.retain_clean_jobs, ) def __substitute_autoscaling_topic_names(self) -> None: diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 3efa0a6f3..005a243dc 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -50,8 +50,8 @@ def inflate(self) -> list[PipelineComponent]: } ), **{ - "config": self._config, - "handlers": self._handlers, + "config": self.config, + "handlers": self.handlers, }, ) inflate_steps.append(kafka_connector) diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 183367a63..8ecfd979d 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -35,8 +35,8 @@ def inflate(self) -> list[PipelineComponent]: } ), **{ - "config": self._config, - "handlers": self._handlers, + "config": self.config, + "handlers": self.handlers, }, ) inflate_steps.append(kafka_connector) From cc6b4a94aab00c3861421e4fbaa1725a3d2b0e2d Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 14:19:54 +0100 Subject: [PATCH 19/44] Exclude schema_type from generate output --- .../base_components/kafka_connect.py | 9 ++++-- .../producer/producer_app.py | 4 +-- .../streams_bootstrap/streams/streams_app.py | 4 +-- kpops/pipeline_generator/pipeline.py | 1 - schema.json | 32 ------------------- 5 files changed, 11 insertions(+), 39 deletions(-) diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 9f29e76ea..710e480b9 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -3,6 +3,7 @@ from abc import ABC from typing import ClassVar, Literal, NoReturn +from pydantic import Field from typing_extensions import override from kpops.cli.pipeline_config import ENV_PREFIX @@ -65,7 +66,9 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): type: ClassVar[str] = "kafka-source-connector" - schema_type: Literal["kafka-source-connector"] = "kafka-source-connector" + schema_type: Literal["kafka-source-connector"] = Field( + "kafka-source-connector", exclude=True + ) @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: @@ -92,7 +95,9 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): type: ClassVar[str] = "kafka-sink-connector" - schema_type: Literal["kafka-sink-connector"] = "kafka-sink-connector" + schema_type: Literal["kafka-sink-connector"] = Field( + "kafka-sink-connector", exclude=True + ) @override def add_input_topics(self, topics: list[str]) -> None: diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 4a6654ca8..3e335460d 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,6 +1,6 @@ from typing import ClassVar, Literal -from pydantic import BaseConfig, Extra +from pydantic import BaseConfig, Extra, Field from typing_extensions import override from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig @@ -21,7 +21,7 @@ class ProducerApp(KafkaApp): """ type: ClassVar[str] = "producer" - schema_type: Literal["producer"] = "producer" + schema_type: Literal["producer"] = Field("producer", exclude=True) app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 26d732a85..85ea93cb1 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -2,7 +2,7 @@ from typing import ClassVar, Literal -from pydantic import BaseConfig, Extra +from pydantic import BaseConfig, Extra, Field from typing_extensions import override from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig @@ -17,7 +17,7 @@ class StreamsApp(KafkaApp): """ type: ClassVar[str] = "streams-app" - schema_type: Literal["streams-app"] = "streams-app" + schema_type: Literal["streams-app"] = Field("streams-app", exclude=True) app: StreamsAppConfig class Config(BaseConfig): diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index ae5bf74b7..e53d826d3 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -223,7 +223,6 @@ def substitute_component_specific_variables( component_object: PipelineComponent, pair: dict ) -> dict: # Override component config with component config in pipeline environment definition - pair.pop("handlers") json_object: dict = json.loads( substitute( json.dumps(pair), diff --git a/schema.json b/schema.json index 62730d536..c734f607b 100644 --- a/schema.json +++ b/schema.json @@ -285,14 +285,6 @@ "version": { "title": "Version", "type": "string" - }, - "type": { - "title": "Schema Type", - "default": "streams-app", - "enum": [ - "streams-app" - ], - "type": "string" } }, "required": [ @@ -371,14 +363,6 @@ "version": { "title": "Version", "type": "string" - }, - "type": { - "title": "Schema Type", - "default": "producer", - "enum": [ - "producer" - ], - "type": "string" } }, "required": [ @@ -407,14 +391,6 @@ }, "to": { "$ref": "#/definitions/ToSection" - }, - "type": { - "title": "Schema Type", - "default": "kafka-source-connector", - "enum": [ - "kafka-source-connector" - ], - "type": "string" } }, "required": [ @@ -438,14 +414,6 @@ }, "to": { "$ref": "#/definitions/ToSection" - }, - "type": { - "title": "Schema Type", - "default": "kafka-sink-connector", - "enum": [ - "kafka-sink-connector" - ], - "type": "string" } }, "required": [ From c86ac79b95a573bc8e9a9d25d7537c255b3bde6c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 14:25:23 +0100 Subject: [PATCH 20/44] Fix producer tests --- tests/components/test_producer_app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index f1939577a..20b8147e0 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -49,7 +49,6 @@ def producer_app( ) -> ProducerApp: return ProducerApp( **{ - "type": "producer-app", "name": self.PRODUCER_APP_NAME, "config": config, "handlers": handlers, @@ -72,7 +71,6 @@ def producer_app( def test_output_topics(self, config: PipelineConfig, handlers: ComponentHandlers): producer_app = ProducerApp( **{ - "type": "producer-app", "name": self.PRODUCER_APP_NAME, "config": config, "handlers": handlers, From c5a2a8e664ac360e228a25dfa0fc6b0130e88bca Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 14:35:52 +0100 Subject: [PATCH 21/44] Fix test --- .../base_components/base_defaults_component.py | 4 ++-- tests/pipeline/test_components/components.py | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index c537a2958..e1ea15d68 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -4,7 +4,7 @@ from collections import deque from collections.abc import Mapping, Sequence from pathlib import Path -from typing import ClassVar, TypeVar +from typing import Any, ClassVar, TypeVar import typer from pydantic import BaseConfig, BaseModel, Field @@ -31,7 +31,7 @@ def __init__(self, **kwargs) -> None: kwargs = self.extend_with_defaults(kwargs) super().__init__(**kwargs) - def extend_with_defaults(self, kwargs) -> dict: + def extend_with_defaults(self, kwargs: dict[str, Any]) -> dict: """ Merges tmp_defaults with all tmp_defaults for parent classes diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 005a243dc..8c335ec6b 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -3,7 +3,6 @@ from schema_registry.client.schema import AvroSchema from typing_extensions import override -from kpops.component_handlers.kafka_connect.model import KafkaConnectConfig from kpops.component_handlers.schema_handler.schema_provider import ( Schema, SchemaProvider, @@ -43,12 +42,10 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - app=KafkaConnectConfig( - **{ - "topics": topic_name, - "transforms.changeTopic.replacement": f"{topic_name}-index-v1", - } - ), + app={ # type: ignore + "topics": topic_name, + "transforms.changeTopic.replacement": f"{topic_name}-index-v1", + }, **{ "config": self.config, "handlers": self.handlers, From 1a8737f7a5d1e6a68270482366f32e2dc8cdb5c9 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 14:43:46 +0100 Subject: [PATCH 22/44] Use field init instead of kwargs --- .../base_defaults_component.py | 6 +-- .../base_components/kafka_connect.py | 4 +- .../producer/producer_app.py | 2 +- .../streams_bootstrap/streams/streams_app.py | 2 +- .../test_base_defaults_component.py | 10 ++-- tests/components/test_kafka_app.py | 2 - tests/components/test_kafka_connect.py | 48 ++++++++++++------- tests/components/test_producer_app.py | 12 ++--- tests/components/test_streams_app.py | 26 ++++------ tests/pipeline/test_components/components.py | 6 +-- .../components.py | 6 +-- 11 files changed, 63 insertions(+), 61 deletions(-) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index e1ea15d68..83275a21e 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -17,11 +17,11 @@ class BaseDefaultsComponent(BaseModel): - type: ClassVar[str] = Field(..., const=True) + type: ClassVar[str] = Field(default=..., const=True) enrich: bool = Field(default=False, exclude=True) - config: PipelineConfig = Field(..., exclude=True) - handlers: ComponentHandlers = Field(..., exclude=True) + config: PipelineConfig = Field(default=..., exclude=True) + handlers: ComponentHandlers = Field(default=..., exclude=True) class Config(BaseConfig): arbitrary_types_allowed = True diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 710e480b9..1a94e5596 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -67,7 +67,7 @@ def clean(self, dry_run: bool) -> None: class KafkaSourceConnector(KafkaConnector): type: ClassVar[str] = "kafka-source-connector" schema_type: Literal["kafka-source-connector"] = Field( - "kafka-source-connector", exclude=True + default="kafka-source-connector", exclude=True ) @override @@ -96,7 +96,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): type: ClassVar[str] = "kafka-sink-connector" schema_type: Literal["kafka-sink-connector"] = Field( - "kafka-sink-connector", exclude=True + default="kafka-sink-connector", exclude=True ) @override diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 3e335460d..eaa2ad05c 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -21,7 +21,7 @@ class ProducerApp(KafkaApp): """ type: ClassVar[str] = "producer" - schema_type: Literal["producer"] = Field("producer", exclude=True) + schema_type: Literal["producer"] = Field(default="producer", exclude=True) app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 85ea93cb1..17f63d4e1 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -17,7 +17,7 @@ class StreamsApp(KafkaApp): """ type: ClassVar[str] = "streams-app" - schema_type: Literal["streams-app"] = Field("streams-app", exclude=True) + schema_type: Literal["streams-app"] = Field(default="streams-app", exclude=True) app: StreamsAppConfig class Config(BaseConfig): diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index 400cb062c..bfd866d1f 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -57,7 +57,7 @@ class TestBaseDefaultsComponent: def test_inherit_defaults( self, config: PipelineConfig, handlers: ComponentHandlers ): - component = TestChildModel(**{"config": config, "handlers": handlers}) + component = TestChildModel(config=config, handlers=handlers) assert ( component.name == "fake-child-name" @@ -77,10 +77,10 @@ def test_inherit_defaults( def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): component = TestChildModel( + config=config, + handlers=handlers, **{ "name": "name-defined-in-pipeline_generator", - "config": config, - "handlers": handlers, }, ) @@ -103,7 +103,7 @@ def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): def test_multiple_generations( self, config: PipelineConfig, handlers: ComponentHandlers ): - component = TestGrandChildModel(**{"config": config, "handlers": handlers}) + component = TestGrandChildModel(config=config, handlers=handlers) assert ( component.name == "fake-child-name" @@ -130,7 +130,7 @@ class TestEnvVarModel(BaseDefaultsComponent): name: str | None = None os.environ["pipeline_name"] = str(DEFAULTS_PATH) - component = TestEnvVarModel(**{"config": config, "handlers": handlers}) + component = TestEnvVarModel(config=config, handlers=handlers) assert component.name == str( DEFAULTS_PATH diff --git a/tests/components/test_kafka_app.py b/tests/components/test_kafka_app.py index 2d2935c16..cf6b00182 100644 --- a/tests/components/test_kafka_app.py +++ b/tests/components/test_kafka_app.py @@ -33,7 +33,6 @@ def test_default_brokers(self, config: PipelineConfig, handlers: ComponentHandle handlers=handlers, config=config, **{ - "type": "streams-app", "name": "example-name", "app": { "namespace": "test-namespace", @@ -55,7 +54,6 @@ def test_should_deploy_kafka_app( handlers=handlers, config=config, **{ - "type": "streams-app", "name": "example-name", "app": { "namespace": "test-namespace", diff --git a/tests/components/test_kafka_connect.py b/tests/components/test_kafka_connect.py index bbbf45746..b41fe3fe5 100644 --- a/tests/components/test_kafka_connect.py +++ b/tests/components/test_kafka_connect.py @@ -57,7 +57,8 @@ def test_connector_config_parsing( connector = KafkaSinkConnector( name="test-connector", app=KafkaConnectConfig(**{"topics": topic_name}), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert getattr(connector.app, "topics") == topic_name @@ -65,7 +66,8 @@ def test_connector_config_parsing( connector = KafkaSinkConnector( name="test-connector", app=KafkaConnectConfig(**{"topics.regex": topic_pattern}), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert getattr(connector.app, "topics.regex") == topic_pattern @@ -77,7 +79,8 @@ def test_connector_config_parsing( "${error_topic_name}": TopicConfig(type=OutputTopicTypes.ERROR), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert ( getattr(connector.app, "errors.deadletterqueue.topic.name") @@ -98,7 +101,8 @@ def test_from_section_parsing_input_topic( topic2: FromTopic(type=InputTopicTypes.INPUT), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert getattr(connector.app, "topics") == f"{topic1},{topic2}" @@ -116,7 +120,8 @@ def test_from_section_parsing_input_pattern( from_=FromSection( topics={topic_pattern: FromTopic(type=InputTopicTypes.INPUT_PATTERN)} ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert getattr(connector.app, "topics.regex") == topic_pattern @@ -136,7 +141,8 @@ def test_deploy_order( ), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) mock_create_topics = mocker.patch.object( @@ -177,7 +183,8 @@ def test_destroy( ), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) mock_destroy_connector = mocker.patch.object( @@ -207,7 +214,8 @@ def test_clean( ), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) mock_delete_topics = mocker.patch.object( @@ -251,7 +259,8 @@ def test_reset( ), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) mock_delete_topics = mocker.patch.object( @@ -281,7 +290,8 @@ def test_clean_without_to( connector = KafkaSinkConnector( name="test-connector", app=KafkaConnectConfig(), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) mock_delete_topics = mocker.patch.object( @@ -316,7 +326,8 @@ def test_from_section_raises_exception( "connector-topic": FromTopic(type=InputTopicTypes.INPUT), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) def test_deploy_order( @@ -335,7 +346,8 @@ def test_deploy_order( ), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) mock_create_topics = mocker.patch.object( @@ -380,7 +392,8 @@ def test_destroy( ), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert connector.handlers.connector_handler @@ -414,7 +427,8 @@ def test_clean( ), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert connector.handlers.connector_handler @@ -456,7 +470,8 @@ def test_clean_without_to( connector = KafkaSourceConnector( name="test-connector", app=KafkaConnectConfig(), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert connector.to is None @@ -499,7 +514,8 @@ def test_reset( ), } ), - **{"config": config, "handlers": handlers}, + config=config, + handlers=handlers, ) assert connector.handlers.connector_handler diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 20b8147e0..88480862f 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -48,10 +48,10 @@ def producer_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> ProducerApp: return ProducerApp( + name=self.PRODUCER_APP_NAME, + config=config, + handlers=handlers, **{ - "name": self.PRODUCER_APP_NAME, - "config": config, - "handlers": handlers, "version": "2.4.2", "app": { "namespace": "test-namespace", @@ -70,10 +70,10 @@ def producer_app( def test_output_topics(self, config: PipelineConfig, handlers: ComponentHandlers): producer_app = ProducerApp( + name=self.PRODUCER_APP_NAME, + config=config, + handlers=handlers, **{ - "name": self.PRODUCER_APP_NAME, - "config": config, - "handlers": handlers, "app": { "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index a1d21dce0..d38a5dd70 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -50,10 +50,9 @@ def streams_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> StreamsApp: return StreamsApp( - handlers=handlers, config=config, + handlers=handlers, **{ - "type": "streams-app", "name": self.STREAMS_APP_NAME, "version": "2.4.2", "app": { @@ -75,11 +74,10 @@ class AnotherType(StreamsApp): type: ClassVar[str] = "test" streams_app = AnotherType( + config=config, + handlers=handlers, **{ - "type": "test", "name": self.STREAMS_APP_NAME, - "config": config, - "handlers": handlers, "app": { "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, @@ -115,10 +113,9 @@ def test_no_empty_input_topic( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - handlers=handlers, config=config, + handlers=handlers, **{ - "type": "test", "name": self.STREAMS_APP_NAME, "app": { "namespace": "test-namespace", @@ -147,10 +144,9 @@ def test_no_empty_input_topic( def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandlers): with pytest.raises(ValueError): StreamsApp( - handlers=handlers, config=config, + handlers=handlers, **{ - "type": "streams-app", "name": self.STREAMS_APP_NAME, "app": { "namespace": "test-namespace", @@ -168,10 +164,9 @@ def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandle with pytest.raises(ValueError): StreamsApp( - handlers=handlers, config=config, + handlers=handlers, **{ - "type": "streams-app", "name": self.STREAMS_APP_NAME, "app": { "namespace": "test-namespace", @@ -185,10 +180,9 @@ def test_set_streams_output_from_to( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - handlers=handlers, config=config, + handlers=handlers, **{ - "type": "streams-app", "name": self.STREAMS_APP_NAME, "app": { "namespace": "test-namespace", @@ -227,10 +221,9 @@ def test_weave_inputs_from_prev_component( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - handlers=handlers, config=config, + handlers=handlers, **{ - "type": "streams-app", "name": self.STREAMS_APP_NAME, "app": { "namespace": "test-namespace", @@ -263,10 +256,9 @@ def test_deploy_order( mocker: MockerFixture, ): streams_app = StreamsApp( - handlers=handlers, config=config, + handlers=handlers, **{ - "type": "streams-app", "name": self.STREAMS_APP_NAME, "version": "2.4.2", "app": { diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 8c335ec6b..64fef5d04 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -46,10 +46,8 @@ def inflate(self) -> list[PipelineComponent]: "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", }, - **{ - "config": self.config, - "handlers": self.handlers, - }, + config=self.config, + handlers=self.handlers, ) inflate_steps.append(kafka_connector) diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 8ecfd979d..d8819deea 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -34,10 +34,8 @@ def inflate(self) -> list[PipelineComponent]: "transforms.changeTopic.replacement": f"{topic_name}-index-v1", } ), - **{ - "config": self.config, - "handlers": self.handlers, - }, + config=self.config, + handlers=self.handlers, ) inflate_steps.append(kafka_connector) From 5949b99a39ddfcac353ae599082149c15e5e608e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 14:51:53 +0100 Subject: [PATCH 23/44] Fix pydantic workaround --- gen_schema.py | 2 +- .../base_defaults_component.py | 8 +++-- schema.json | 32 +++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/gen_schema.py b/gen_schema.py index 2dc67269e..b46fac84a 100644 --- a/gen_schema.py +++ b/gen_schema.py @@ -16,7 +16,7 @@ # adapted from https://github.com/tiangolo/fastapi/issues/1378#issuecomment-764966955 def field_schema(field: ModelField, **kwargs: Any) -> Any: - if field.field_info.exclude: + if field.field_info.extra.get("hidden_from_schema"): raise SkipField(f"{field.name} field is being hidden") else: return original_field_schema(field, **kwargs) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 83275a21e..c64deb6cb 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -19,9 +19,11 @@ class BaseDefaultsComponent(BaseModel): type: ClassVar[str] = Field(default=..., const=True) - enrich: bool = Field(default=False, exclude=True) - config: PipelineConfig = Field(default=..., exclude=True) - handlers: ComponentHandlers = Field(default=..., exclude=True) + enrich: bool = Field(default=False, exclude=True, hidden_from_schema=True) + config: PipelineConfig = Field(default=..., exclude=True, hidden_from_schema=True) + handlers: ComponentHandlers = Field( + default=..., exclude=True, hidden_from_schema=True + ) class Config(BaseConfig): arbitrary_types_allowed = True diff --git a/schema.json b/schema.json index c734f607b..62730d536 100644 --- a/schema.json +++ b/schema.json @@ -285,6 +285,14 @@ "version": { "title": "Version", "type": "string" + }, + "type": { + "title": "Schema Type", + "default": "streams-app", + "enum": [ + "streams-app" + ], + "type": "string" } }, "required": [ @@ -363,6 +371,14 @@ "version": { "title": "Version", "type": "string" + }, + "type": { + "title": "Schema Type", + "default": "producer", + "enum": [ + "producer" + ], + "type": "string" } }, "required": [ @@ -391,6 +407,14 @@ }, "to": { "$ref": "#/definitions/ToSection" + }, + "type": { + "title": "Schema Type", + "default": "kafka-source-connector", + "enum": [ + "kafka-source-connector" + ], + "type": "string" } }, "required": [ @@ -414,6 +438,14 @@ }, "to": { "$ref": "#/definitions/ToSection" + }, + "type": { + "title": "Schema Type", + "default": "kafka-sink-connector", + "enum": [ + "kafka-sink-connector" + ], + "type": "string" } }, "required": [ From 59ab0c7d4bbf9c737260e18e7a45a9a1948bfe1b Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:32:43 +0100 Subject: [PATCH 24/44] Update schema --- .../base_components/kafka_connect.py | 4 +- .../base_components/kubernetes_app.py | 2 +- .../producer/producer_app.py | 4 +- .../streams_bootstrap/streams/streams_app.py | 4 +- schema.json | 192 ++++++++++++++++-- 5 files changed, 187 insertions(+), 19 deletions(-) diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 4929b9c58..29438625d 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -203,7 +203,7 @@ def __uninstall_connect_resetter(self, release_name: str, dry_run: bool) -> None class KafkaSourceConnector(KafkaConnector): type: ClassVar[str] = "kafka-source-connector" schema_type: Literal["kafka-source-connector"] = Field( - default="kafka-source-connector", exclude=True + default="kafka-source-connector", alias="schema_type", exclude=True ) offset_topic: str | None = None @@ -233,7 +233,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): type: ClassVar[str] = "kafka-sink-connector" schema_type: Literal["kafka-sink-connector"] = Field( - default="kafka-sink-connector", exclude=True + default="kafka-sink-connector", alias="schema_type", exclude=True ) @override diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 187304b26..e545e9e31 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -32,7 +32,7 @@ class Config(CamelCaseConfig): # TODO: label and annotations class KubernetesApp(PipelineComponent): - """Base kubernetes app""" + """Base Kubernetes app""" type: ClassVar[str] = "kubernetes-app" app: KubernetesAppConfig diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 85be72ca4..2fdbfc692 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -22,7 +22,9 @@ class ProducerApp(KafkaApp): """ type: ClassVar[str] = "producer" - schema_type: Literal["producer"] = Field(default="producer", exclude=True) + schema_type: Literal["producer"] = Field( + default="producer", alias="schema_type", exclude=True + ) app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index b133555f3..0b0c3347d 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -16,7 +16,9 @@ class StreamsApp(KafkaApp): """ type: ClassVar[str] = "streams-app" - schema_type: Literal["streams-app"] = Field(default="streams-app", exclude=True) + schema_type: Literal["streams-app"] = Field( + default="streams-app", alias="schema_type", exclude=True + ) app: StreamsAppConfig class Config(BaseConfig): diff --git a/schema.json b/schema.json index 62730d536..eee48c00d 100644 --- a/schema.json +++ b/schema.json @@ -171,10 +171,6 @@ "description": "StreamsBoostrap app configurations.\n\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", "type": "object", "properties": { - "namespace": { - "title": "Namespace", - "type": "string" - }, "streams": { "$ref": "#/definitions/StreamsConfig" }, @@ -187,7 +183,6 @@ } }, "required": [ - "namespace", "streams" ] }, @@ -264,6 +259,62 @@ "topics" ] }, + "RepoAuthFlags": { + "title": "RepoAuthFlags", + "type": "object", + "properties": { + "username": { + "title": "Username", + "type": "string" + }, + "password": { + "title": "Password", + "type": "string" + }, + "caFile": { + "title": "Cafile", + "type": "string", + "format": "path" + }, + "insecureSkipTlsVerify": { + "title": "Insecureskiptlsverify", + "default": false, + "type": "boolean" + } + } + }, + "HelmRepoConfig": { + "title": "HelmRepoConfig", + "type": "object", + "properties": { + "repositoryName": { + "title": "Repositoryname", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + }, + "repoAuthFlags": { + "title": "Repoauthflags", + "default": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + }, + "allOf": [ + { + "$ref": "#/definitions/RepoAuthFlags" + } + ] + } + }, + "required": [ + "repositoryName", + "url" + ] + }, "StreamsApp": { "title": "StreamsApp", "description": "StreamsApp component that configures a streams bootstrap app", @@ -282,8 +333,31 @@ "to": { "$ref": "#/definitions/ToSection" }, + "repoConfig": { + "title": "Repoconfig", + "default": { + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/", + "repo_auth_flags": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + } + }, + "allOf": [ + { + "$ref": "#/definitions/HelmRepoConfig" + } + ] + }, + "namespace": { + "title": "Namespace", + "type": "string" + }, "version": { "title": "Version", + "default": "2.7.0", "type": "string" }, "type": { @@ -297,7 +371,8 @@ }, "required": [ "name", - "app" + "app", + "namespace" ] }, "ProducerStreamsConfig": { @@ -333,10 +408,6 @@ "title": "ProducerValues", "type": "object", "properties": { - "namespace": { - "title": "Namespace", - "type": "string" - }, "streams": { "$ref": "#/definitions/ProducerStreamsConfig" }, @@ -346,7 +417,6 @@ } }, "required": [ - "namespace", "streams" ] }, @@ -368,8 +438,31 @@ "to": { "$ref": "#/definitions/ToSection" }, + "repoConfig": { + "title": "Repoconfig", + "default": { + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/", + "repo_auth_flags": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + } + }, + "allOf": [ + { + "$ref": "#/definitions/HelmRepoConfig" + } + ] + }, + "namespace": { + "title": "Namespace", + "type": "string" + }, "version": { "title": "Version", + "default": "2.7.0", "type": "string" }, "type": { @@ -383,7 +476,8 @@ }, "required": [ "name", - "app" + "app", + "namespace" ] }, "KafkaConnectConfig": { @@ -408,6 +502,38 @@ "to": { "$ref": "#/definitions/ToSection" }, + "repoConfig": { + "title": "Repoconfig", + "default": { + "repository_name": "bakdata-kafka-connect-resetter", + "url": "https://bakdata.github.io/kafka-connect-resetter/", + "repo_auth_flags": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + } + }, + "allOf": [ + { + "$ref": "#/definitions/HelmRepoConfig" + } + ] + }, + "namespace": { + "title": "Namespace", + "type": "string" + }, + "version": { + "title": "Version", + "default": "1.0.4", + "type": "string" + }, + "resetterValues": { + "title": "Resettervalues", + "description": "Overriding Kafka Connect Resetter Helm values. E.g. to override the Image Tag etc.", + "type": "object" + }, "type": { "title": "Schema Type", "default": "kafka-source-connector", @@ -415,11 +541,16 @@ "kafka-source-connector" ], "type": "string" + }, + "offsetTopic": { + "title": "Offsettopic", + "type": "string" } }, "required": [ "name", - "app" + "app", + "namespace" ] }, "KafkaSinkConnector": { @@ -439,6 +570,38 @@ "to": { "$ref": "#/definitions/ToSection" }, + "repoConfig": { + "title": "Repoconfig", + "default": { + "repository_name": "bakdata-kafka-connect-resetter", + "url": "https://bakdata.github.io/kafka-connect-resetter/", + "repo_auth_flags": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + } + }, + "allOf": [ + { + "$ref": "#/definitions/HelmRepoConfig" + } + ] + }, + "namespace": { + "title": "Namespace", + "type": "string" + }, + "version": { + "title": "Version", + "default": "1.0.4", + "type": "string" + }, + "resetterValues": { + "title": "Resettervalues", + "description": "Overriding Kafka Connect Resetter Helm values. E.g. to override the Image Tag etc.", + "type": "object" + }, "type": { "title": "Schema Type", "default": "kafka-sink-connector", @@ -450,7 +613,8 @@ }, "required": [ "name", - "app" + "app", + "namespace" ] } } From 344eced341de4da825068e0b9f7da4f733b55a82 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:33:28 +0100 Subject: [PATCH 25/44] Cosmetic --- kpops/components/base_components/kafka_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index af6401376..15859d1fd 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -37,7 +37,7 @@ class KafkaAppConfig(KubernetesAppConfig): class KafkaApp(KubernetesApp): """ - Base component for kafka-based components. + Base component for Kafka-based components. Producer or streaming apps should inherit from this class. """ From 00af95b00ded01f7fdaf55e02e37854e2409ff22 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:38:15 +0100 Subject: [PATCH 26/44] Fix test name --- ...{test_kafka_connect_sink.py => test_kafka_sink_connector.py} | 2 +- ...t_kafka_connect_source.py => test_kafka_source_connector.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/components/{test_kafka_connect_sink.py => test_kafka_sink_connector.py} (99%) rename tests/components/{test_kafka_connect_source.py => test_kafka_source_connector.py} (99%) diff --git a/tests/components/test_kafka_connect_sink.py b/tests/components/test_kafka_sink_connector.py similarity index 99% rename from tests/components/test_kafka_connect_sink.py rename to tests/components/test_kafka_sink_connector.py index 8fa1e351b..b72f2fde1 100644 --- a/tests/components/test_kafka_connect_sink.py +++ b/tests/components/test_kafka_sink_connector.py @@ -32,7 +32,7 @@ CONNECTOR_CLEAN_NAME = "test-connector-with-long-name-0123456789abcdef-clean" -class TestKafkaConnectorSink: +class TestKafkaSinkConnector: @pytest.fixture def log_info_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.components.base_components.kafka_connect.log.info") diff --git a/tests/components/test_kafka_connect_source.py b/tests/components/test_kafka_source_connector.py similarity index 99% rename from tests/components/test_kafka_connect_source.py rename to tests/components/test_kafka_source_connector.py index cd4e14320..cbff009bf 100644 --- a/tests/components/test_kafka_connect_source.py +++ b/tests/components/test_kafka_source_connector.py @@ -29,7 +29,7 @@ CONNECTOR_CLEAN_NAME = "test-connector-with-long-name-0123456789abcdef-clean" -class TestKafkaConnectorSource: +class TestKafkaSourceConnector: @pytest.fixture def config(slef) -> PipelineConfig: return PipelineConfig( From b261c3331b1899220dc8ea255172713f32f40b5c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:39:31 +0100 Subject: [PATCH 27/44] Cleanup test --- tests/components/test_kafka_sink_connector.py | 24 +++++++++---------- .../components/test_kafka_source_connector.py | 12 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/components/test_kafka_sink_connector.py b/tests/components/test_kafka_sink_connector.py index b72f2fde1..090c639c3 100644 --- a/tests/components/test_kafka_sink_connector.py +++ b/tests/components/test_kafka_sink_connector.py @@ -69,9 +69,9 @@ def test_connector_config_parsing( topic_name = "connector-topic" connector = KafkaSinkConnector( name="test-connector", - app=KafkaConnectConfig(**{"topics": topic_name}), config=config, handlers=handlers, + app=KafkaConnectConfig(**{"topics": topic_name}), namespace="test-namespace", ) assert getattr(connector.app, "topics") == topic_name @@ -79,15 +79,17 @@ def test_connector_config_parsing( topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", - app=KafkaConnectConfig(**{"topics.regex": topic_pattern}), config=config, handlers=handlers, + app=KafkaConnectConfig(**{"topics.regex": topic_pattern}), namespace="test-namespace", ) assert getattr(connector.app, "topics.regex") == topic_pattern connector = KafkaSinkConnector( name="test-connector", + config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", to=ToSection( @@ -95,8 +97,6 @@ def test_connector_config_parsing( "${error_topic_name}": TopicConfig(type=OutputTopicTypes.ERROR), } ), - config=config, - handlers=handlers, ) assert ( getattr(connector.app, "errors.deadletterqueue.topic.name") @@ -110,6 +110,8 @@ def test_from_section_parsing_input_topic( topic2 = "connector-topic2" connector = KafkaSinkConnector( name="test-connector", + config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", from_=FromSection( @@ -118,8 +120,6 @@ def test_from_section_parsing_input_topic( topic2: FromTopic(type=InputTopicTypes.INPUT), } ), - config=config, - handlers=handlers, ) assert getattr(connector.app, "topics") == f"{topic1},{topic2}" @@ -133,13 +133,13 @@ def test_from_section_parsing_input_pattern( topic_pattern = ".*" connector = KafkaSinkConnector( name="test-connector", + config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", from_=FromSection( topics={topic_pattern: FromTopic(type=InputTopicTypes.INPUT_PATTERN)} ), - config=config, - handlers=handlers, ) assert getattr(connector.app, "topics.regex") == topic_pattern @@ -151,6 +151,8 @@ def test_deploy_order( ): connector = KafkaSinkConnector( name="test-connector", + config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", to=ToSection( @@ -160,8 +162,6 @@ def test_deploy_order( ), } ), - config=config, - handlers=handlers, ) mock_create_topics = mocker.patch.object( @@ -194,6 +194,8 @@ def test_destroy( ): connector = KafkaSinkConnector( name="test-connector", + config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", to=ToSection( @@ -203,8 +205,6 @@ def test_destroy( ), } ), - config=config, - handlers=handlers, ) mock_destroy_connector = mocker.patch.object( diff --git a/tests/components/test_kafka_source_connector.py b/tests/components/test_kafka_source_connector.py index cbff009bf..8cce42d58 100644 --- a/tests/components/test_kafka_source_connector.py +++ b/tests/components/test_kafka_source_connector.py @@ -63,8 +63,8 @@ def test_from_section_raises_exception( with pytest.raises(NotImplementedError): KafkaSourceConnector( name="test-connector", - handlers=handlers, config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", from_=FromSection( @@ -82,8 +82,8 @@ def test_deploy_order( ): connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", to=ToSection( @@ -129,8 +129,8 @@ def test_destroy( ] = "kafka-connect-offsets" connector = KafkaSourceConnector( name="test-connector", - handlers=handlers, config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", to=ToSection( @@ -163,8 +163,8 @@ def test_reset( ): connector = KafkaSourceConnector( name=CONNECTOR_NAME, - handlers=handlers, config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", offset_topic="kafka-connect-offsets", @@ -242,8 +242,8 @@ def test_clean( ): connector = KafkaSourceConnector( name=CONNECTOR_NAME, - handlers=handlers, config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", offset_topic="kafka-connect-offsets", @@ -321,8 +321,8 @@ def test_clean_without_to( ): connector = KafkaSourceConnector( name=CONNECTOR_NAME, - handlers=handlers, config=config, + handlers=handlers, app=KafkaConnectConfig(), namespace="test-namespace", offset_topic="kafka-connect-offsets", From f2589755c2f48f0fb79a66f888e519963f1f0c2b Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:39:31 +0100 Subject: [PATCH 28/44] Cleanup test --- tests/components/test_kubernetes_app.py | 30 ++++++++++++------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index 739417ed0..76294bd25 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -66,10 +66,10 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( ): kubernetes_app = KubernetesApp( - app=app_value, + name="test-kubernetes-apps", config=config, handlers=handlers, - name="test-kubernetes-apps", + app=app_value, namespace="test-namespace", ) @@ -100,10 +100,10 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( ): repo_config = HelmRepoConfig(repository_name="test-repo", url="mock://test") kubernetes_app = KubernetesApp( - app=app_value, + name="test-kubernetes-apps", config=config, handlers=handlers, - name="test-kubernetes-apps", + app=app_value, namespace="test-namespace", repo_config=repo_config, version="3.4.5", @@ -146,10 +146,10 @@ def test_should_print_helm_diff_after_install_when_dry_run_and_helm_diff_enabled ): kubernetes_app = KubernetesApp( - app=app_value, + name="test-kubernetes-apps", config=config, handlers=handlers, - name="test-kubernetes-apps", + app=app_value, namespace="test-namespace", ) mocker.patch.object( @@ -169,10 +169,10 @@ def test_should_raise_not_implemented_error_when_helm_chart_is_not_set( ): kubernetes_app = KubernetesApp( - app=app_value, + name="test-kubernetes-apps", config=config, handlers=handlers, - name="test-kubernetes-apps", + app=app_value, namespace="test-namespace", ) @@ -193,10 +193,10 @@ def test_should_call_helm_uninstall_when_destroying_kubernetes_app( ): kubernetes_app = KubernetesApp( - app=app_value, + name="test-kubernetes-apps", config=config, handlers=handlers, - name="test-kubernetes-apps", + app=app_value, namespace="test-namespace", ) @@ -219,25 +219,23 @@ def test_should_raise_value_error_when_name_is_not_valid( ): assert KubernetesApp( - app=app_value, + name="example-component-with-very-long-name-longer-than-most-of-our-kubernetes-apps", config=config, handlers=handlers, - name="example-component-with-very-long-name-longer-than-most-of-our-kubernetes-apps", + app=app_value, namespace="test-namespace", ) with pytest.raises(ValueError): assert KubernetesApp( - type="test", + name="Not-Compatible*", config=config, handlers=handlers, - name="Not-Compatible*", ) with pytest.raises(ValueError): assert KubernetesApp( - type="test", + name="snake_case", config=config, handlers=handlers, - name="snake_case", ) From d96035b2a7f4caf378865d5f335a3cec75a5785d Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:39:31 +0100 Subject: [PATCH 29/44] Cleanup test --- tests/components/test_kafka_app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/components/test_kafka_app.py b/tests/components/test_kafka_app.py index 18d65c777..8a4ae55dc 100644 --- a/tests/components/test_kafka_app.py +++ b/tests/components/test_kafka_app.py @@ -33,10 +33,10 @@ def handlers(self) -> ComponentHandlers: def test_default_configs(self, config: PipelineConfig, handlers: ComponentHandlers): kafka_app = KafkaApp( - handlers=handlers, + name="example-name", config=config, + handlers=handlers, **{ - "name": "example-name", "namespace": "test-namespace", "app": { "streams": { @@ -59,10 +59,10 @@ def test_should_deploy_kafka_app( self, config: PipelineConfig, handlers: ComponentHandlers, mocker: MockerFixture ): kafka_app = KafkaApp( - handlers=handlers, + name="example-name", config=config, + handlers=handlers, **{ - "name": "example-name", "namespace": "test-namespace", "app": { "streams": { From d35259e3135a3bdc6ee122cdf0bb717ee7d6f3c4 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:39:31 +0100 Subject: [PATCH 30/44] Cleanup test --- tests/components/test_pipeline_component.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/components/test_pipeline_component.py b/tests/components/test_pipeline_component.py index a20a940f8..0dbbf3aca 100644 --- a/tests/components/test_pipeline_component.py +++ b/tests/components/test_pipeline_component.py @@ -17,11 +17,8 @@ class TestPipelineComponent: def test_topic_substitution(self): pipeline_component = PipelineComponent( - handlers=ComponentHandlers( - schema_handler=MagicMock(), - connector_handler=MagicMock(), - topic_handler=MagicMock(), - ), + type="plain-pipeline-component", + name="test-pipeline-component", config=pipeline_config.PipelineConfig( defaults_path=DEFAULTS_PATH, environment="development", @@ -30,8 +27,11 @@ def test_topic_substitution(self): default_output_topic_name="output-${component_type}", ), ), - name="test-pipeline-component", - type="plane-pipeline-component", + handlers=ComponentHandlers( + schema_handler=MagicMock(), + connector_handler=MagicMock(), + topic_handler=MagicMock(), + ), to=ToSection( models={}, topics={ @@ -46,6 +46,6 @@ def test_topic_substitution(self): ) assert pipeline_component.to - assert "error-plane-pipeline-component" in pipeline_component.to.topics - assert "output-plane-pipeline-component" in pipeline_component.to.topics + assert "error-plain-pipeline-component" in pipeline_component.to.topics + assert "output-plain-pipeline-component" in pipeline_component.to.topics assert len(pipeline_component.to.topics.keys()) == 2 From 5c278278e268c72312d32e3f66547b5456d8de79 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:39:31 +0100 Subject: [PATCH 31/44] Cleanup test --- tests/components/test_streams_app.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 190abef94..3b3daff2b 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -49,10 +49,10 @@ def streams_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> StreamsApp: return StreamsApp( + name=self.STREAMS_APP_NAME, config=config, handlers=handlers, **{ - "name": self.STREAMS_APP_NAME, "namespace": "test-namespace", "app": { "streams": {"brokers": "fake-broker:9092"}, @@ -69,10 +69,10 @@ def streams_app( def test_set_topics(self, config: PipelineConfig, handlers: ComponentHandlers): streams_app = StreamsApp( + name=self.STREAMS_APP_NAME, config=config, handlers=handlers, **{ - "name": self.STREAMS_APP_NAME, "namespace": "test-namespace", "app": { "streams": {"brokers": "fake-broker:9092"}, @@ -108,10 +108,10 @@ def test_no_empty_input_topic( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( + name=self.STREAMS_APP_NAME, config=config, handlers=handlers, **{ - "name": self.STREAMS_APP_NAME, "namespace": "test-namespace", "app": { "streams": {"brokers": "fake-broker:9092"}, @@ -139,10 +139,10 @@ def test_no_empty_input_topic( def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandlers): with pytest.raises(ValueError): StreamsApp( + name=self.STREAMS_APP_NAME, config=config, handlers=handlers, **{ - "name": self.STREAMS_APP_NAME, "namespace": "test-namespace", "app": { "streams": {"brokers": "fake-broker:9092"}, @@ -159,10 +159,10 @@ def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandle with pytest.raises(ValueError): StreamsApp( + name=self.STREAMS_APP_NAME, config=config, handlers=handlers, **{ - "name": self.STREAMS_APP_NAME, "namespace": "test-namespace", "app": { "streams": {"brokers": "fake-broker:9092"}, @@ -175,10 +175,10 @@ def test_set_streams_output_from_to( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( + name=self.STREAMS_APP_NAME, config=config, handlers=handlers, **{ - "name": self.STREAMS_APP_NAME, "namespace": "test-namespace", "app": { "streams": {"brokers": "fake-broker:9092"}, @@ -216,10 +216,10 @@ def test_weave_inputs_from_prev_component( self, config: PipelineConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( + name=self.STREAMS_APP_NAME, config=config, handlers=handlers, **{ - "name": self.STREAMS_APP_NAME, "namespace": "test-namespace", "app": { "streams": {"brokers": "fake-broker:9092"}, @@ -251,10 +251,10 @@ def test_deploy_order( mocker: MockerFixture, ): streams_app = StreamsApp( + name=self.STREAMS_APP_NAME, config=config, handlers=handlers, **{ - "name": self.STREAMS_APP_NAME, "namespace": "test-namespace", "app": { "streams": {"brokers": "fake-broker:9092"}, From d0bff0b5d7e7df0bb94dc7fbac12a15ece81cbca Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:39:31 +0100 Subject: [PATCH 32/44] Cleanup test --- .../test_components_without_schema_handler/components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 321f73bef..5c7e7a48d 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -28,9 +28,9 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - namespace="example-namespace", config=self.config, handlers=self.handlers, + namespace="example-namespace", app=KafkaConnectConfig( # TODO **{ "topics": topic_name, From 382f27ed26a4460d1b869a30070318552d1a10dc Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:39:31 +0100 Subject: [PATCH 33/44] Cleanup test --- tests/components/test_base_defaults_component.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index bfd866d1f..569e915a1 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -79,9 +79,7 @@ def test_inherit(self, config: PipelineConfig, handlers: ComponentHandlers): component = TestChildModel( config=config, handlers=handlers, - **{ - "name": "name-defined-in-pipeline_generator", - }, + **{"name": "name-defined-in-pipeline_generator"}, ) assert ( From b38edb99aa15f5d3c4e60768029e459c8bb0fe52 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:49:34 +0100 Subject: [PATCH 34/44] Mark todo --- tests/pipeline/test_components/components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 48a481780..4fda2078d 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -45,7 +45,7 @@ def inflate(self) -> list[PipelineComponent]: config=self.config, handlers=self.handlers, namespace="example-namespace", - app={ # type: ignore + app={ # type: ignore # FIXME "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", }, From 9d9559fd8b3a0a82f6e3c07a116624db47e1cfe8 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:50:02 +0100 Subject: [PATCH 35/44] Remove another todo --- .../test_components_without_schema_handler/components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 5c7e7a48d..c899ffbda 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -31,7 +31,7 @@ def inflate(self) -> list[PipelineComponent]: config=self.config, handlers=self.handlers, namespace="example-namespace", - app=KafkaConnectConfig( # TODO + app=KafkaConnectConfig( **{ "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", From ea43c6071029d3ec642bbbd362d0b7ee9be0d77e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:51:02 +0100 Subject: [PATCH 36/44] Remove base class in schema gen --- gen_schema.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/gen_schema.py b/gen_schema.py index b46fac84a..d48bf718c 100644 --- a/gen_schema.py +++ b/gen_schema.py @@ -24,13 +24,7 @@ def field_schema(field: ModelField, **kwargs: Any) -> Any: schema.field_schema = field_schema -ComponentType = ( - StreamsApp - | ProducerApp - | KafkaSourceConnector - | KafkaSinkConnector - # | PipelineComponent -) +ComponentType = StreamsApp | ProducerApp | KafkaSourceConnector | KafkaSinkConnector AnnotatedPipelineComponent = Annotated[ From ebe8d9604fdf46d9c7d9d4794fe8c9d28360837c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 26 Jan 2023 16:52:50 +0100 Subject: [PATCH 37/44] Rename variable --- gen_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gen_schema.py b/gen_schema.py index d48bf718c..621ba62ea 100644 --- a/gen_schema.py +++ b/gen_schema.py @@ -24,11 +24,11 @@ def field_schema(field: ModelField, **kwargs: Any) -> Any: schema.field_schema = field_schema -ComponentType = StreamsApp | ProducerApp | KafkaSourceConnector | KafkaSinkConnector +PipelineComponent = StreamsApp | ProducerApp | KafkaSourceConnector | KafkaSinkConnector AnnotatedPipelineComponent = Annotated[ - ComponentType, Field(discriminator="schema_type") + PipelineComponent, Field(discriminator="schema_type") ] schema = schema_json_of( From 6ff157a93e2b8ebdff86013b85346e9f2dc74ff0 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 30 Jan 2023 09:48:24 +0100 Subject: [PATCH 38/44] Exclude `schema_type` from camel case conversion --- kpops/components/base_components/kafka_connect.py | 4 ++-- kpops/components/streams_bootstrap/producer/producer_app.py | 4 +--- kpops/components/streams_bootstrap/streams/streams_app.py | 4 +--- kpops/utils/pydantic.py | 2 ++ 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/kpops/components/base_components/kafka_connect.py b/kpops/components/base_components/kafka_connect.py index 29438625d..4929b9c58 100644 --- a/kpops/components/base_components/kafka_connect.py +++ b/kpops/components/base_components/kafka_connect.py @@ -203,7 +203,7 @@ def __uninstall_connect_resetter(self, release_name: str, dry_run: bool) -> None class KafkaSourceConnector(KafkaConnector): type: ClassVar[str] = "kafka-source-connector" schema_type: Literal["kafka-source-connector"] = Field( - default="kafka-source-connector", alias="schema_type", exclude=True + default="kafka-source-connector", exclude=True ) offset_topic: str | None = None @@ -233,7 +233,7 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): type: ClassVar[str] = "kafka-sink-connector" schema_type: Literal["kafka-sink-connector"] = Field( - default="kafka-sink-connector", alias="schema_type", exclude=True + default="kafka-sink-connector", exclude=True ) @override diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 2fdbfc692..85be72ca4 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -22,9 +22,7 @@ class ProducerApp(KafkaApp): """ type: ClassVar[str] = "producer" - schema_type: Literal["producer"] = Field( - default="producer", alias="schema_type", exclude=True - ) + schema_type: Literal["producer"] = Field(default="producer", exclude=True) app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 0b0c3347d..b133555f3 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -16,9 +16,7 @@ class StreamsApp(KafkaApp): """ type: ClassVar[str] = "streams-app" - schema_type: Literal["streams-app"] = Field( - default="streams-app", alias="schema_type", exclude=True - ) + schema_type: Literal["streams-app"] = Field(default="streams-app", exclude=True) app: StreamsAppConfig class Config(BaseConfig): diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index 3791c205f..3118177a9 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -3,6 +3,8 @@ def to_camel(field: str) -> str: + if field == "schema_type": + return field return humps.camelize(field) # type: ignore From 51c54c883e6974abebb066cc1916ab22a265212a Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 30 Jan 2023 10:03:51 +0100 Subject: [PATCH 39/44] Fix Poetry lockfile --- poetry.lock | 1142 ++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 572 insertions(+), 572 deletions(-) diff --git a/poetry.lock b/poetry.lock index 92c49376d..b56b3e7f9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + [[package]] name = "aiofiles" version = "22.1.0" @@ -5,6 +7,10 @@ description = "File support for asyncio." category = "main" optional = false python-versions = ">=3.7,<4.0" +files = [ + {file = "aiofiles-22.1.0-py3-none-any.whl", hash = "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad"}, + {file = "aiofiles-22.1.0.tar.gz", hash = "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"}, +] [[package]] name = "anyio" @@ -13,6 +19,10 @@ description = "High level compatibility layer for multiple asynchronous event lo category = "main" optional = false python-versions = ">=3.6.2" +files = [ + {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, + {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, +] [package.dependencies] idna = ">=2.8" @@ -30,6 +40,10 @@ description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] [package.extras] dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] @@ -44,6 +58,10 @@ description = "Screen-scraping library" category = "dev" optional = false python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, + {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, +] [package.dependencies] soupsieve = ">1.2" @@ -59,6 +77,20 @@ description = "The uncompromising code formatter." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, + {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, + {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, + {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, + {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, + {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, + {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, + {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, + {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, + {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, + {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, + {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, +] [package.dependencies] click = ">=8.0.0" @@ -80,6 +112,10 @@ description = "Extensible memoizing collections and decorators" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "cachetools-5.2.0-py3-none-any.whl", hash = "sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db"}, + {file = "cachetools-5.2.0.tar.gz", hash = "sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757"}, +] [[package]] name = "certifi" @@ -88,6 +124,10 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] [[package]] name = "cfgv" @@ -96,6 +136,10 @@ description = "Validate configuration and produce human readable error messages. category = "dev" optional = false python-versions = ">=3.6.1" +files = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] [[package]] name = "charset-normalizer" @@ -104,6 +148,10 @@ description = "The Real First Universal Charset Detector. Open, modern and activ category = "main" optional = false python-versions = ">=3.6.0" +files = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] [package.extras] unicode-backport = ["unicodedata2"] @@ -115,6 +163,10 @@ description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -126,6 +178,10 @@ description = "Cross-platform colored terminal text." category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "commonmark" @@ -134,6 +190,10 @@ description = "Python parser for the CommonMark Markdown spec" category = "main" optional = false python-versions = "*" +files = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] [package.extras] test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] @@ -145,6 +205,10 @@ description = "Dictdiffer is a library that helps you to diff and patch dictiona category = "main" optional = false python-versions = "*" +files = [ + {file = "dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595"}, + {file = "dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578"}, +] [package.extras] all = ["Sphinx (>=3)", "check-manifest (>=0.42)", "mock (>=1.3.0)", "numpy (>=1.13.0)", "numpy (>=1.15.0)", "numpy (>=1.18.0)", "numpy (>=1.20.0)", "pytest (==5.4.3)", "pytest (>=6)", "pytest-cov (>=2.10.1)", "pytest-isort (>=1.2.0)", "pytest-pycodestyle (>=2)", "pytest-pycodestyle (>=2.2.0)", "pytest-pydocstyle (>=2)", "pytest-pydocstyle (>=2.2.0)", "sphinx (>=3)", "sphinx-rtd-theme (>=0.2)", "tox (>=3.7.0)"] @@ -159,6 +223,10 @@ description = "Distribution utilities" category = "dev" optional = false python-versions = "*" +files = [ + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, +] [[package]] name = "exceptiongroup" @@ -167,6 +235,10 @@ description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, + {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, +] [package.extras] test = ["pytest (>=6)"] @@ -178,6 +250,29 @@ description = "Fast read/write of AVRO files" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, + {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, + {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, + {file = "fastavro-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:54b60e79506a456bcfc940560fa2c73a7a8e3ddc58a1ae4d94fdd99f6b02aef0"}, + {file = "fastavro-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4da6d83abd04a804310667f8d1fc23d44e9ed91ed9a9bc9c1fcd906e0c403b12"}, + {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c664302f94064adeb93403c61c74e07b9710526403eba3b59169f99bb99c55c"}, + {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aab162f02e64bf82d0282430a3c6ec7a36982b1c5d479e7dcc278e6d62a84b8"}, + {file = "fastavro-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:63d0d2a2bb3e85d006c834633be51b105a50b0dc7cc8423b06f30601b532adf4"}, + {file = "fastavro-1.7.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a07a3245049529d6d9028d664361cc4990e74d035d2303875295e2f7b97eba09"}, + {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df57d31cb58770a9066790250e9f4ec91242c69e1dc62ea732a6fb2406a8f96"}, + {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b905634b5763ca08c9f7b51486e2c3ae7907f5d9bc48208c69b16ccbe8455e90"}, + {file = "fastavro-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:749206f1cec3d7429546e49db5708f4571497d35181b6b334c4844133f230515"}, + {file = "fastavro-1.7.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a0ebd5c1269085179de4b3f072de274fb66a471ecbc5245bd8684e6f94943c2f"}, + {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e35b94b692f8cca0096c89abf1937efed66252dea0b3b3165babfb3c289fb7"}, + {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202424a5a7831b773f0f2cc2f82e89ed1726051fd5994f13dc678514144e10d4"}, + {file = "fastavro-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:601f8c2ec166bd721f4b12eafe195dd1373d3f8dce4fe2425abd2df3e3968ac7"}, + {file = "fastavro-1.7.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:91cc9be740abca894082af4fe3ab9db057d4e5fa783cfa9a94c02ec041bf4346"}, + {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4463b00242e4baf52d499aeefab46a26d9dd18d808d4219cd4d21089da540e"}, + {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7348d318858203bd108e6bcde177d8a6f0590b52bc624d815f26fb6c37029bb"}, + {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, + {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, +] [package.extras] codecs = ["lz4", "python-snappy", "zstandard"] @@ -192,6 +287,10 @@ description = "A fast native implementation of diff algorithm with a pure python category = "dev" optional = false python-versions = "*" +files = [ + {file = "fastdiff-0.3.0-py2.py3-none-any.whl", hash = "sha256:ca5f61f6ddf5a1564ddfd98132ad28e7abe4a88a638a8b014a2214f71e5918ec"}, + {file = "fastdiff-0.3.0.tar.gz", hash = "sha256:4dfa09c47832a8c040acda3f1f55fc0ab4d666f0e14e6951e6da78d59acd945a"}, +] [package.dependencies] wasmer = ">=1.0.0" @@ -204,6 +303,10 @@ description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, + {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, +] [package.extras] docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] @@ -216,6 +319,10 @@ description = "the modular source code checker: pep8 pyflakes and co" category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, + {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, +] [package.dependencies] mccabe = ">=0.6.0,<0.7.0" @@ -229,6 +336,10 @@ description = "Copy your docs directly to the gh-pages branch." category = "dev" optional = false python-versions = "*" +files = [ + {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, + {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, +] [package.dependencies] python-dateutil = ">=2.8.1" @@ -243,6 +354,10 @@ description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] [[package]] name = "httpcore" @@ -251,6 +366,10 @@ description = "A minimal low-level HTTP client." category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "httpcore-0.16.2-py3-none-any.whl", hash = "sha256:52c79095197178856724541e845f2db86d5f1527640d9254b5b8f6f6cebfdee6"}, + {file = "httpcore-0.16.2.tar.gz", hash = "sha256:c35c5176dc82db732acfd90b581a3062c999a72305df30c0fc8fafd8e4aca068"}, +] [package.dependencies] anyio = ">=3.0,<5.0" @@ -269,6 +388,10 @@ description = "The next generation HTTP client." category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "httpx-0.23.1-py3-none-any.whl", hash = "sha256:0b9b1f0ee18b9978d637b0776bfd7f54e2ca278e063e3586d8f01cda89e042a8"}, + {file = "httpx-0.23.1.tar.gz", hash = "sha256:202ae15319be24efe9a8bd4ed4360e68fde7b38bcc2ce87088d416f026667d19"}, +] [package.dependencies] certifi = "*" @@ -289,6 +412,10 @@ description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "identify-2.5.10-py2.py3-none-any.whl", hash = "sha256:fb7c2feaeca6976a3ffa31ec3236a6911fbc51aec9acc111de2aed99f244ade2"}, + {file = "identify-2.5.10.tar.gz", hash = "sha256:dce9e31fee7dbc45fea36a9e855c316b8fbf807e65a862f160840bb5a2bf5dfd"}, +] [package.extras] license = ["ukkonen"] @@ -300,6 +427,10 @@ description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] [[package]] name = "iniconfig" @@ -308,18 +439,26 @@ description = "iniconfig: brain-dead simple config-ini parsing" category = "dev" optional = false python-versions = "*" +files = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] [[package]] name = "isort" -version = "5.11.3" +version = "5.12.0" description = "A Python utility / library to sort Python imports." category = "dev" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, + {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, +] [package.extras] -colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile-deprecated-finder = ["pipreqs", "requirementslib"] +colors = ["colorama (>=0.4.3)"] +pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] plugins = ["setuptools"] requirements-deprecated-finder = ["pip-api", "pipreqs"] @@ -330,6 +469,10 @@ description = "A very fast and expressive template engine." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] [package.dependencies] MarkupSafe = ">=2.0" @@ -344,6 +487,10 @@ description = "An implementation of JSON Schema validation for Python" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, + {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, +] [package.dependencies] attrs = ">=17.4.0" @@ -360,6 +507,10 @@ description = "Python implementation of Markdown." category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "Markdown-3.3.7-py3-none-any.whl", hash = "sha256:f5da449a6e1c989a4cea2631aa8ee67caa5a2ef855d551c88f9e309f4634c621"}, + {file = "Markdown-3.3.7.tar.gz", hash = "sha256:cbb516f16218e643d8e0a95b309f77eb118cb138d39a4f27851e6a63581db874"}, +] [package.extras] testing = ["coverage", "pyyaml"] @@ -371,6 +522,48 @@ description = "Safely add untrusted strings to HTML/XML markup." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-win32.whl", hash = "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-win32.whl", hash = "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-win32.whl", hash = "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-win32.whl", hash = "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, + {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, +] [[package]] name = "mccabe" @@ -379,6 +572,10 @@ description = "McCabe checker, plugin for flake8" category = "dev" optional = false python-versions = "*" +files = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] [[package]] name = "mergedeep" @@ -387,6 +584,10 @@ description = "A deep merge function for 🐍." category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, + {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, +] [[package]] name = "mike" @@ -395,6 +596,10 @@ description = "Manage multiple versions of your MkDocs-powered documentation" category = "dev" optional = false python-versions = "*" +files = [ + {file = "mike-1.1.2-py3-none-any.whl", hash = "sha256:4c307c28769834d78df10f834f57f810f04ca27d248f80a75f49c6fa2d1527ca"}, + {file = "mike-1.1.2.tar.gz", hash = "sha256:56c3f1794c2d0b5fdccfa9b9487beb013ca813de2e3ad0744724e9d34d40b77b"}, +] [package.dependencies] jinja2 = "*" @@ -413,6 +618,10 @@ description = "Project documentation with Markdown." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "mkdocs-1.4.2-py3-none-any.whl", hash = "sha256:c8856a832c1e56702577023cd64cc5f84948280c1c0fcc6af4cd39006ea6aa8c"}, + {file = "mkdocs-1.4.2.tar.gz", hash = "sha256:8947af423a6d0facf41ea1195b8e1e8c85ad94ac95ae307fe11232e0424b11c5"}, +] [package.dependencies] click = ">=7.0" @@ -437,6 +646,9 @@ description = "MkDocs plugin supports image lightbox with GLightbox." category = "dev" optional = false python-versions = "*" +files = [ + {file = "mkdocs-glightbox-0.3.1.tar.gz", hash = "sha256:ac85e2d4d422cc4a670fa276840f0aa3064a1ec4ad25ccb6d6e82d11bb11e513"}, +] [package.dependencies] beautifulsoup4 = ">=4.11.1" @@ -448,6 +660,10 @@ description = "Unleash the power of MkDocs with macros and variables" category = "dev" optional = false python-versions = ">=3.5" +files = [ + {file = "mkdocs-macros-plugin-0.7.0.tar.gz", hash = "sha256:9e64e1cabcf6925359de29fe54f62d5847fb455c2528c440b87f8f1240650608"}, + {file = "mkdocs_macros_plugin-0.7.0-py3-none-any.whl", hash = "sha256:96bdabeb98b96139544f0048ea2f5cb80c7befde6b21e94c6d4596c22774cbcf"}, +] [package.dependencies] jinja2 = "*" @@ -466,6 +682,10 @@ description = "Documentation that simply works" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "mkdocs_material-8.5.11-py3-none-any.whl", hash = "sha256:c907b4b052240a5778074a30a78f31a1f8ff82d7012356dc26898b97559f082e"}, + {file = "mkdocs_material-8.5.11.tar.gz", hash = "sha256:b0ea0513fd8cab323e8a825d6692ea07fa83e917bb5db042e523afecc7064ab7"}, +] [package.dependencies] jinja2 = ">=3.0.2" @@ -483,6 +703,10 @@ description = "Extension pack for Python Markdown and MkDocs Material." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "mkdocs_material_extensions-1.1.1-py3-none-any.whl", hash = "sha256:e41d9f38e4798b6617ad98ca8f7f1157b1e4385ac1459ca1e4ea219b556df945"}, + {file = "mkdocs_material_extensions-1.1.1.tar.gz", hash = "sha256:9c003da71e2cc2493d910237448c672e00cefc800d3d6ae93d2fc69979e3bd93"}, +] [[package]] name = "mypy" @@ -491,6 +715,38 @@ description = "Optional static typing for Python" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "mypy-0.990-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:aaf1be63e0207d7d17be942dcf9a6b641745581fe6c64df9a38deb562a7dbafa"}, + {file = "mypy-0.990-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d555aa7f44cecb7ea3c0ac69d58b1a5afb92caa017285a8e9c4efbf0518b61b4"}, + {file = "mypy-0.990-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f694d6d09a460b117dccb6857dda269188e3437c880d7b60fa0014fa872d1e9"}, + {file = "mypy-0.990-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:269f0dfb6463b8780333310ff4b5134425157ef0d2b1d614015adaf6d6a7eabd"}, + {file = "mypy-0.990-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8798c8ed83aa809f053abff08664bdca056038f5a02af3660de00b7290b64c47"}, + {file = "mypy-0.990-cp310-cp310-win_amd64.whl", hash = "sha256:47a9955214615108c3480a500cfda8513a0b1cd3c09a1ed42764ca0dd7b931dd"}, + {file = "mypy-0.990-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4a8a6c10f4c63fbf6ad6c03eba22c9331b3946a4cec97f008e9ffb4d3b31e8e2"}, + {file = "mypy-0.990-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cd2dd3730ba894ec2a2082cc703fbf3e95a08479f7be84912e3131fc68809d46"}, + {file = "mypy-0.990-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7da0005e47975287a92b43276e460ac1831af3d23032c34e67d003388a0ce8d0"}, + {file = "mypy-0.990-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:262c543ef24deb10470a3c1c254bb986714e2b6b1a67d66daf836a548a9f316c"}, + {file = "mypy-0.990-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3ff201a0c6d3ea029d73b1648943387d75aa052491365b101f6edd5570d018ea"}, + {file = "mypy-0.990-cp311-cp311-win_amd64.whl", hash = "sha256:1767830da2d1afa4e62b684647af0ff79b401f004d7fa08bc5b0ce2d45bcd5ec"}, + {file = "mypy-0.990-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6826d9c4d85bbf6d68cb279b561de6a4d8d778ca8e9ab2d00ee768ab501a9852"}, + {file = "mypy-0.990-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46897755f944176fbc504178422a5a2875bbf3f7436727374724842c0987b5af"}, + {file = "mypy-0.990-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0680389c34284287fe00e82fc8bccdea9aff318f7e7d55b90d967a13a9606013"}, + {file = "mypy-0.990-cp37-cp37m-win_amd64.whl", hash = "sha256:b08541a06eed35b543ae1a6b301590eb61826a1eb099417676ddc5a42aa151c5"}, + {file = "mypy-0.990-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:be88d665e76b452c26fb2bdc3d54555c01226fba062b004ede780b190a50f9db"}, + {file = "mypy-0.990-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b8f4a8213b1fd4b751e26b59ae0e0c12896568d7e805861035c7a15ed6dc9eb"}, + {file = "mypy-0.990-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b6f85c2ad378e3224e017904a051b26660087b3b76490d533b7344f1546d3ff"}, + {file = "mypy-0.990-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ee5f99817ee70254e7eb5cf97c1b11dda29c6893d846c8b07bce449184e9466"}, + {file = "mypy-0.990-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49082382f571c3186ce9ea0bd627cb1345d4da8d44a8377870f4442401f0a706"}, + {file = "mypy-0.990-cp38-cp38-win_amd64.whl", hash = "sha256:aba38e3dd66bdbafbbfe9c6e79637841928ea4c79b32e334099463c17b0d90ef"}, + {file = "mypy-0.990-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9d851c09b981a65d9d283a8ccb5b1d0b698e580493416a10942ef1a04b19fd37"}, + {file = "mypy-0.990-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d847dd23540e2912d9667602271e5ebf25e5788e7da46da5ffd98e7872616e8e"}, + {file = "mypy-0.990-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cc6019808580565040cd2a561b593d7c3c646badd7e580e07d875eb1bf35c695"}, + {file = "mypy-0.990-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a3150d409609a775c8cb65dbe305c4edd7fe576c22ea79d77d1454acd9aeda8"}, + {file = "mypy-0.990-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3227f14fe943524f5794679156488f18bf8d34bfecd4623cf76bc55958d229c5"}, + {file = "mypy-0.990-cp39-cp39-win_amd64.whl", hash = "sha256:c76c769c46a1e6062a84837badcb2a7b0cdb153d68601a61f60739c37d41cc74"}, + {file = "mypy-0.990-py3-none-any.whl", hash = "sha256:8f1940325a8ed460ba03d19ab83742260fa9534804c317224e5d4e5aa588e2d6"}, + {file = "mypy-0.990.tar.gz", hash = "sha256:72382cb609142dba3f04140d016c94b4092bc7b4d98ca718740dc989e5271b8d"}, +] [package.dependencies] mypy-extensions = ">=0.4.3" @@ -510,6 +766,10 @@ description = "Experimental type system extensions for programs checked with the category = "dev" optional = false python-versions = "*" +files = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] [[package]] name = "nodeenv" @@ -518,6 +778,10 @@ description = "Node.js virtual environment builder" category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, +] [package.dependencies] setuptools = "*" @@ -529,6 +793,10 @@ description = "Core utilities for Python packages" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, + {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, +] [[package]] name = "pathspec" @@ -537,6 +805,10 @@ description = "Utility library for gitignore style pattern matching of file path category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pathspec-0.10.3-py3-none-any.whl", hash = "sha256:3c95343af8b756205e2aba76e843ba9520a24dd84f68c22b9f93251507509dd6"}, + {file = "pathspec-0.10.3.tar.gz", hash = "sha256:56200de4077d9d0791465aa9095a01d421861e405b5096955051deefd697d6f6"}, +] [[package]] name = "platformdirs" @@ -545,6 +817,10 @@ description = "A small Python package for determining appropriate platform-speci category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "platformdirs-2.6.0-py3-none-any.whl", hash = "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca"}, + {file = "platformdirs-2.6.0.tar.gz", hash = "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e"}, +] [package.extras] docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] @@ -557,6 +833,10 @@ description = "plugin and hook calling mechanisms for python" category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] [package.extras] dev = ["pre-commit", "tox"] @@ -569,6 +849,10 @@ description = "A framework for managing and maintaining multi-language pre-commi category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] [package.dependencies] cfgv = ">=2.0.0" @@ -585,6 +869,10 @@ description = "Python style guide checker" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, + {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, +] [[package]] name = "pydantic" @@ -593,30 +881,76 @@ description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" - -[package.dependencies] -python-dotenv = {version = ">=0.10.4", optional = true, markers = "extra == \"dotenv\""} -typing-extensions = ">=4.1.0" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - -[[package]] -name = "pyflakes" -version = "2.4.0" -description = "passive checker of Python programs" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pygments" +files = [ + {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, + {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, + {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, + {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, + {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, + {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, + {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, + {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, + {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, +] + +[package.dependencies] +python-dotenv = {version = ">=0.10.4", optional = true, markers = "extra == \"dotenv\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyflakes" +version = "2.4.0" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, + {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, +] + +[[package]] +name = "pygments" version = "2.13.0" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] [package.extras] plugins = ["importlib-metadata"] @@ -628,6 +962,10 @@ description = "🐫 Convert strings (and dictionary keys) between snake case, c category = "main" optional = false python-versions = "*" +files = [ + {file = "pyhumps-3.8.0-py3-none-any.whl", hash = "sha256:060e1954d9069f428232a1adda165db0b9d8dfdce1d265d36df7fbff540acfd6"}, + {file = "pyhumps-3.8.0.tar.gz", hash = "sha256:498026258f7ee1a8e447c2e28526c0bea9407f9a59c03260aee4bd6c04d681a3"}, +] [[package]] name = "pymdown-extensions" @@ -636,6 +974,10 @@ description = "Extension pack for Python Markdown." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pymdown_extensions-9.9-py3-none-any.whl", hash = "sha256:ac698c15265680db5eb13cd4342abfcde2079ac01e5486028f47a1b41547b859"}, + {file = "pymdown_extensions-9.9.tar.gz", hash = "sha256:0f8fb7b74a37a61cc34e90b2c91865458b713ec774894ffad64353a5fce85cfc"}, +] [package.dependencies] markdown = ">=3.2" @@ -647,6 +989,30 @@ description = "Persistent/Functional/Immutable data structures" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "pyrsistent-0.19.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a"}, + {file = "pyrsistent-0.19.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a"}, + {file = "pyrsistent-0.19.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:055ab45d5911d7cae397dc418808d8802fb95262751872c841c170b0dbf51eed"}, + {file = "pyrsistent-0.19.2-cp310-cp310-win32.whl", hash = "sha256:456cb30ca8bff00596519f2c53e42c245c09e1a4543945703acd4312949bfd41"}, + {file = "pyrsistent-0.19.2-cp310-cp310-win_amd64.whl", hash = "sha256:b39725209e06759217d1ac5fcdb510e98670af9e37223985f330b611f62e7425"}, + {file = "pyrsistent-0.19.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aede922a488861de0ad00c7630a6e2d57e8023e4be72d9d7147a9fcd2d30712"}, + {file = "pyrsistent-0.19.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:879b4c2f4d41585c42df4d7654ddffff1239dc4065bc88b745f0341828b83e78"}, + {file = "pyrsistent-0.19.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c43bec251bbd10e3cb58ced80609c5c1eb238da9ca78b964aea410fb820d00d6"}, + {file = "pyrsistent-0.19.2-cp37-cp37m-win32.whl", hash = "sha256:d690b18ac4b3e3cab73b0b7aa7dbe65978a172ff94970ff98d82f2031f8971c2"}, + {file = "pyrsistent-0.19.2-cp37-cp37m-win_amd64.whl", hash = "sha256:3ba4134a3ff0fc7ad225b6b457d1309f4698108fb6b35532d015dca8f5abed73"}, + {file = "pyrsistent-0.19.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a178209e2df710e3f142cbd05313ba0c5ebed0a55d78d9945ac7a4e09d923308"}, + {file = "pyrsistent-0.19.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e371b844cec09d8dc424d940e54bba8f67a03ebea20ff7b7b0d56f526c71d584"}, + {file = "pyrsistent-0.19.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111156137b2e71f3a9936baf27cb322e8024dac3dc54ec7fb9f0bcf3249e68bb"}, + {file = "pyrsistent-0.19.2-cp38-cp38-win32.whl", hash = "sha256:e5d8f84d81e3729c3b506657dddfe46e8ba9c330bf1858ee33108f8bb2adb38a"}, + {file = "pyrsistent-0.19.2-cp38-cp38-win_amd64.whl", hash = "sha256:9cd3e9978d12b5d99cbdc727a3022da0430ad007dacf33d0bf554b96427f33ab"}, + {file = "pyrsistent-0.19.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f1258f4e6c42ad0b20f9cfcc3ada5bd6b83374516cd01c0960e3cb75fdca6770"}, + {file = "pyrsistent-0.19.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21455e2b16000440e896ab99e8304617151981ed40c29e9507ef1c2e4314ee95"}, + {file = "pyrsistent-0.19.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfd880614c6237243ff53a0539f1cb26987a6dc8ac6e66e0c5a40617296a045e"}, + {file = "pyrsistent-0.19.2-cp39-cp39-win32.whl", hash = "sha256:71d332b0320642b3261e9fee47ab9e65872c2bd90260e5d225dabeed93cbd42b"}, + {file = "pyrsistent-0.19.2-cp39-cp39-win_amd64.whl", hash = "sha256:dec3eac7549869365fe263831f576c8457f6c833937c68542d08fde73457d291"}, + {file = "pyrsistent-0.19.2-py3-none-any.whl", hash = "sha256:ea6b79a02a28550c98b6ca9c35b9f492beaa54d7c5c9e9949555893c8a9234d0"}, + {file = "pyrsistent-0.19.2.tar.gz", hash = "sha256:bfa0351be89c9fcbcb8c9879b826f4353be10f58f8a677efab0c017bf7137ec2"}, +] [[package]] name = "pytest" @@ -655,6 +1021,10 @@ description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, + {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, +] [package.dependencies] attrs = ">=19.2.0" @@ -675,6 +1045,10 @@ description = "Thin-wrapper around the mock package for easier use with pytest" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, + {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, +] [package.dependencies] pytest = ">=5.0" @@ -689,6 +1063,10 @@ description = "Extensions to the standard Python datetime module" category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] [package.dependencies] six = ">=1.5" @@ -700,6 +1078,10 @@ description = "Read key-value pairs from a .env file and set them as environment category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "python-dotenv-0.21.0.tar.gz", hash = "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"}, + {file = "python_dotenv-0.21.0-py3-none-any.whl", hash = "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5"}, +] [package.extras] cli = ["click (>=5.0)"] @@ -711,6 +1093,9 @@ description = "Python Rest Client to interact against Schema Registry Confluent category = "main" optional = false python-versions = "*" +files = [ + {file = "python-schema-registry-client-2.4.1.tar.gz", hash = "sha256:1e0562e3c6447a836f09b116cfe4d8904dee4e9d9d17adcfdf3e8916b897de86"}, +] [package.dependencies] aiofiles = ">=0.7.0" @@ -730,6 +1115,48 @@ description = "YAML parser and emitter for Python" category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] [[package]] name = "pyyaml-env-tag" @@ -738,6 +1165,10 @@ description = "A custom YAML tag for referencing environment variables in YAML f category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "pyyaml_env_tag-0.1-py3-none-any.whl", hash = "sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069"}, + {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, +] [package.dependencies] pyyaml = "*" @@ -749,6 +1180,10 @@ description = "Python HTTP for Humans." category = "main" optional = false python-versions = ">=3.7, <4" +files = [ + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, +] [package.dependencies] certifi = ">=2017.4.17" @@ -767,6 +1202,10 @@ description = "A utility library for mocking out the `requests` Python library." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, + {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, +] [package.dependencies] requests = ">=2.22.0,<3.0" @@ -784,6 +1223,10 @@ description = "Validating URI References per RFC 3986" category = "main" optional = false python-versions = "*" +files = [ + {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, + {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, +] [package.dependencies] idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} @@ -798,6 +1241,10 @@ description = "Render rich text, tables, progress bars, syntax highlighting, mar category = "main" optional = false python-versions = ">=3.6.3,<4.0.0" +files = [ + {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, + {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, +] [package.dependencies] commonmark = ">=0.9.0,<0.10.0" @@ -813,6 +1260,10 @@ description = "Easily download, build, install, upgrade, and uninstall Python pa category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, + {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, +] [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] @@ -826,6 +1277,10 @@ description = "Tool to Detect Surrounding Shell" category = "main" optional = false python-versions = ">=3.4" +files = [ + {file = "shellingham-1.5.0-py2.py3-none-any.whl", hash = "sha256:a8f02ba61b69baaa13facdba62908ca8690a94b8119b69f5ec5873ea85f7391b"}, + {file = "shellingham-1.5.0.tar.gz", hash = "sha256:72fb7f5c63103ca2cb91b23dee0c71fe8ad6fbfd46418ef17dbe40db51592dad"}, +] [[package]] name = "six" @@ -834,6 +1289,10 @@ description = "Python 2 and 3 compatibility utilities" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] [[package]] name = "snapshottest" @@ -842,6 +1301,10 @@ description = "Snapshot testing for pytest, unittest, Django, and Nose" category = "dev" optional = false python-versions = "*" +files = [ + {file = "snapshottest-0.6.0-py2.py3-none-any.whl", hash = "sha256:9b177cffe0870c589df8ddbee0a770149c5474b251955bdbde58b7f32a4ec429"}, + {file = "snapshottest-0.6.0.tar.gz", hash = "sha256:bbcaf81d92d8e330042e5c928e13d9f035e99e91b314fe55fda949c2f17b653c"}, +] [package.dependencies] fastdiff = ">=0.1.4,<1" @@ -860,6 +1323,10 @@ description = "Sniff out which async library your code is running under" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] [[package]] name = "soupsieve" @@ -868,6 +1335,10 @@ description = "A modern CSS selector implementation for Beautiful Soup." category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"}, + {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, +] [[package]] name = "termcolor" @@ -876,6 +1347,10 @@ description = "ANSI color formatting for output in terminal" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "termcolor-2.1.1-py3-none-any.whl", hash = "sha256:fa852e957f97252205e105dd55bbc23b419a70fec0085708fc0515e399f304fd"}, + {file = "termcolor-2.1.1.tar.gz", hash = "sha256:67cee2009adc6449c650f6bcf3bdeed00c8ba53a8cda5362733c53e0a39fb70b"}, +] [package.extras] tests = ["pytest", "pytest-cov"] @@ -887,6 +1362,10 @@ description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] [[package]] name = "tomli" @@ -895,6 +1374,10 @@ description = "A lil' TOML parser" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] [[package]] name = "typer" @@ -903,6 +1386,10 @@ description = "Typer, build great CLIs. Easy to code. Based on Python type hints category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, + {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, +] [package.dependencies] click = ">=7.1.1,<9.0.0" @@ -923,6 +1410,10 @@ description = "Typing stubs for toml" category = "dev" optional = false python-versions = "*" +files = [ + {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, + {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, +] [[package]] name = "typing-extensions" @@ -931,6 +1422,10 @@ description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] [[package]] name = "urllib3" @@ -939,6 +1434,10 @@ description = "HTTP library with thread-safe connection pooling, file post, and category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, + {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, +] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] @@ -952,6 +1451,10 @@ description = "Flexible version handling" category = "dev" optional = false python-versions = "*" +files = [ + {file = "verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31"}, + {file = "verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e"}, +] [package.extras] test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] @@ -963,6 +1466,10 @@ description = "Virtual Python Environment builder" category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, + {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, +] [package.dependencies] distlib = ">=0.3.6,<1" @@ -980,6 +1487,22 @@ description = "Python extension to run WebAssembly binaries" category = "dev" optional = false python-versions = "*" +files = [ + {file = "wasmer-1.1.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:c2af4b907ae2dabcac41e316e811d5937c93adf1f8b05c5d49427f8ce0f37630"}, + {file = "wasmer-1.1.0-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:ab1ae980021e5ec0bf0c6cdd3b979b1d15a5f3eb2b8a32da8dcb1156e4a1e484"}, + {file = "wasmer-1.1.0-cp310-none-win_amd64.whl", hash = "sha256:d0d93aec6215893d33e803ef0a8d37bf948c585dd80ba0e23a83fafee820bc03"}, + {file = "wasmer-1.1.0-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:1e63d16bd6e2e2272d8721647831de5c537e0bb08002ee6d7abf167ec02d5178"}, + {file = "wasmer-1.1.0-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:85e6a5bf44853e8e6a12e947ee3412da9e84f7ce49fc165ba5dbd293e9c5c405"}, + {file = "wasmer-1.1.0-cp37-none-win_amd64.whl", hash = "sha256:a182a6eca9b46d895b4985fc822fab8da3d2f84fab74ca27e55a7430a7fcf336"}, + {file = "wasmer-1.1.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:214d9a3cfb577ea9449eb2b5f13adceae34c55365e4c3d930066beb86a7f67bc"}, + {file = "wasmer-1.1.0-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:b9e5605552bd7d2bc6337519b176defe83bc69b98abf3caaaefa4f7ec231d18a"}, + {file = "wasmer-1.1.0-cp38-none-win_amd64.whl", hash = "sha256:20b5190112e2e94a8947967f2bc683c9685855d0f34130d8434c87a55216a3bd"}, + {file = "wasmer-1.1.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:ee442f0970f40ec5e32011c92fd753fb2061da0faa13de13fafc730c31be34e3"}, + {file = "wasmer-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa112198b743cff2e391230436813fb4b244a24443e37866522b7197e3a034da"}, + {file = "wasmer-1.1.0-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:c0b37117f6d3ff51ee96431c7d224d99799b08d174e30fcd0fcd7e2e3cb8140c"}, + {file = "wasmer-1.1.0-cp39-none-win_amd64.whl", hash = "sha256:a0a4730ec4907a4cb0d9d4a77ea2608c2c814f22a22b73fc80be0f110e014836"}, + {file = "wasmer-1.1.0-py3-none-any.whl", hash = "sha256:2caf8c67feae9cd4246421551036917811c446da4f27ad4c989521ef42751931"}, +] [[package]] name = "wasmer-compiler-cranelift" @@ -988,6 +1511,22 @@ description = "The Cranelift compiler for the `wasmer` package (to compile WebAs category = "dev" optional = false python-versions = "*" +files = [ + {file = "wasmer_compiler_cranelift-1.1.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:9869910179f39696a020edc5689f7759257ac1cce569a7a0fcf340c59788baad"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:405546ee864ac158a4107f374dfbb1c8d6cfb189829bdcd13050143a4bd98f28"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp310-none-win_amd64.whl", hash = "sha256:bdf75af9ef082e6aeb752550f694273340ece970b65099e0746db0f972760d11"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:7d9c782b7721789b16e303b7e70c59df370896dd62b77e2779e3a44b4e1aa20c"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:ff7dd5bd69030b63521c24583bf0f5457cd2580237340b91ce35370f72a4a1cc"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp37-none-win_amd64.whl", hash = "sha256:447285402e366a34667a674db70458c491acd6940b797c175c0b0027f48e64bb"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:55a524985179f6b7b88ac973e8fac5a2574d3b125a966fba75fedd5a2525e484"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:bd03db5a916ead51b442c66acad38847dfe127cf90b2019b1680f1920c4f8d06"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp38-none-win_amd64.whl", hash = "sha256:157d87cbd1d04adbad55b50cb4bedc28e444caf74797fd96df17390667e58699"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:ff25fc99ebafa04a6c271d08a90d17b927930e3019a2b333c7cfb48ba32c6f71"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9697ae082317a56776df8ff7df8c922eac38488ef38d3478fe5f0ca144c185ab"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:2a4349b1ddd727bd46bc5ede741839dcfc5153c52f064a83998c4150d5d4a85c"}, + {file = "wasmer_compiler_cranelift-1.1.0-cp39-none-win_amd64.whl", hash = "sha256:32fe38614fccc933da77ee4372904a5fa9c12b859209a2e4048a8284c4c371f2"}, + {file = "wasmer_compiler_cranelift-1.1.0-py3-none-any.whl", hash = "sha256:200fea80609cfb088457327acf66d5aa61f4c4f66b5a71133ada960b534c7355"}, +] [[package]] name = "watchdog" @@ -996,554 +1535,7 @@ description = "Filesystem events monitoring" category = "dev" optional = false python-versions = ">=3.6" - -[package.extras] -watchmedo = ["PyYAML (>=3.10)"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.10" -content-hash = "a83e67a82f0dd0f8cb53b33e2ce96261dbc4ccd452627923e82c1ab8ef7e4d73" - -[metadata.files] -aiofiles = [ - {file = "aiofiles-22.1.0-py3-none-any.whl", hash = "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad"}, - {file = "aiofiles-22.1.0.tar.gz", hash = "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"}, -] -anyio = [ - {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, - {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] -beautifulsoup4 = [ - {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, - {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, -] -black = [ - {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, - {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, - {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, - {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, - {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, - {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, - {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, - {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, - {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, - {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, - {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, - {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, -] -cachetools = [ - {file = "cachetools-5.2.0-py3-none-any.whl", hash = "sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db"}, - {file = "cachetools-5.2.0.tar.gz", hash = "sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757"}, -] -certifi = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] -cfgv = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -commonmark = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] -dictdiffer = [ - {file = "dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595"}, - {file = "dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578"}, -] -distlib = [ - {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, - {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, -] -exceptiongroup = [ - {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, - {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, -] -fastavro = [ - {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, - {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, - {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, - {file = "fastavro-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:54b60e79506a456bcfc940560fa2c73a7a8e3ddc58a1ae4d94fdd99f6b02aef0"}, - {file = "fastavro-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4da6d83abd04a804310667f8d1fc23d44e9ed91ed9a9bc9c1fcd906e0c403b12"}, - {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c664302f94064adeb93403c61c74e07b9710526403eba3b59169f99bb99c55c"}, - {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aab162f02e64bf82d0282430a3c6ec7a36982b1c5d479e7dcc278e6d62a84b8"}, - {file = "fastavro-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:63d0d2a2bb3e85d006c834633be51b105a50b0dc7cc8423b06f30601b532adf4"}, - {file = "fastavro-1.7.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a07a3245049529d6d9028d664361cc4990e74d035d2303875295e2f7b97eba09"}, - {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df57d31cb58770a9066790250e9f4ec91242c69e1dc62ea732a6fb2406a8f96"}, - {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b905634b5763ca08c9f7b51486e2c3ae7907f5d9bc48208c69b16ccbe8455e90"}, - {file = "fastavro-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:749206f1cec3d7429546e49db5708f4571497d35181b6b334c4844133f230515"}, - {file = "fastavro-1.7.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a0ebd5c1269085179de4b3f072de274fb66a471ecbc5245bd8684e6f94943c2f"}, - {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e35b94b692f8cca0096c89abf1937efed66252dea0b3b3165babfb3c289fb7"}, - {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202424a5a7831b773f0f2cc2f82e89ed1726051fd5994f13dc678514144e10d4"}, - {file = "fastavro-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:601f8c2ec166bd721f4b12eafe195dd1373d3f8dce4fe2425abd2df3e3968ac7"}, - {file = "fastavro-1.7.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:91cc9be740abca894082af4fe3ab9db057d4e5fa783cfa9a94c02ec041bf4346"}, - {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4463b00242e4baf52d499aeefab46a26d9dd18d808d4219cd4d21089da540e"}, - {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7348d318858203bd108e6bcde177d8a6f0590b52bc624d815f26fb6c37029bb"}, - {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, - {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, -] -fastdiff = [ - {file = "fastdiff-0.3.0-py2.py3-none-any.whl", hash = "sha256:ca5f61f6ddf5a1564ddfd98132ad28e7abe4a88a638a8b014a2214f71e5918ec"}, - {file = "fastdiff-0.3.0.tar.gz", hash = "sha256:4dfa09c47832a8c040acda3f1f55fc0ab4d666f0e14e6951e6da78d59acd945a"}, -] -filelock = [ - {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, - {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, -] -flake8 = [ - {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, - {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, -] -ghp-import = [ - {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, - {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, -] -h11 = [ - {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, - {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, -] -httpcore = [ - {file = "httpcore-0.16.2-py3-none-any.whl", hash = "sha256:52c79095197178856724541e845f2db86d5f1527640d9254b5b8f6f6cebfdee6"}, - {file = "httpcore-0.16.2.tar.gz", hash = "sha256:c35c5176dc82db732acfd90b581a3062c999a72305df30c0fc8fafd8e4aca068"}, -] -httpx = [ - {file = "httpx-0.23.1-py3-none-any.whl", hash = "sha256:0b9b1f0ee18b9978d637b0776bfd7f54e2ca278e063e3586d8f01cda89e042a8"}, - {file = "httpx-0.23.1.tar.gz", hash = "sha256:202ae15319be24efe9a8bd4ed4360e68fde7b38bcc2ce87088d416f026667d19"}, -] -identify = [ - {file = "identify-2.5.10-py2.py3-none-any.whl", hash = "sha256:fb7c2feaeca6976a3ffa31ec3236a6911fbc51aec9acc111de2aed99f244ade2"}, - {file = "identify-2.5.10.tar.gz", hash = "sha256:dce9e31fee7dbc45fea36a9e855c316b8fbf807e65a862f160840bb5a2bf5dfd"}, -] -idna = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] -iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] -isort = [ - {file = "isort-5.11.3-py3-none-any.whl", hash = "sha256:83155ffa936239d986b0f190347a3f2285f42a9b9e1725c89d865b27dd0627e5"}, - {file = "isort-5.11.3.tar.gz", hash = "sha256:a8ca25fbfad0f7d5d8447a4314837298d9f6b23aed8618584c894574f626b64b"}, -] -jinja2 = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] -jsonschema = [ - {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, - {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, -] -markdown = [ - {file = "Markdown-3.3.7-py3-none-any.whl", hash = "sha256:f5da449a6e1c989a4cea2631aa8ee67caa5a2ef855d551c88f9e309f4634c621"}, - {file = "Markdown-3.3.7.tar.gz", hash = "sha256:cbb516f16218e643d8e0a95b309f77eb118cb138d39a4f27851e6a63581db874"}, -] -markupsafe = [ - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win32.whl", hash = "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win32.whl", hash = "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win32.whl", hash = "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win32.whl", hash = "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, - {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, -] -mccabe = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, -] -mergedeep = [ - {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, - {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, -] -mike = [ - {file = "mike-1.1.2-py3-none-any.whl", hash = "sha256:4c307c28769834d78df10f834f57f810f04ca27d248f80a75f49c6fa2d1527ca"}, - {file = "mike-1.1.2.tar.gz", hash = "sha256:56c3f1794c2d0b5fdccfa9b9487beb013ca813de2e3ad0744724e9d34d40b77b"}, -] -mkdocs = [ - {file = "mkdocs-1.4.2-py3-none-any.whl", hash = "sha256:c8856a832c1e56702577023cd64cc5f84948280c1c0fcc6af4cd39006ea6aa8c"}, - {file = "mkdocs-1.4.2.tar.gz", hash = "sha256:8947af423a6d0facf41ea1195b8e1e8c85ad94ac95ae307fe11232e0424b11c5"}, -] -mkdocs-glightbox = [ - {file = "mkdocs-glightbox-0.3.1.tar.gz", hash = "sha256:ac85e2d4d422cc4a670fa276840f0aa3064a1ec4ad25ccb6d6e82d11bb11e513"}, -] -mkdocs-macros-plugin = [ - {file = "mkdocs-macros-plugin-0.7.0.tar.gz", hash = "sha256:9e64e1cabcf6925359de29fe54f62d5847fb455c2528c440b87f8f1240650608"}, - {file = "mkdocs_macros_plugin-0.7.0-py3-none-any.whl", hash = "sha256:96bdabeb98b96139544f0048ea2f5cb80c7befde6b21e94c6d4596c22774cbcf"}, -] -mkdocs-material = [ - {file = "mkdocs_material-8.5.11-py3-none-any.whl", hash = "sha256:c907b4b052240a5778074a30a78f31a1f8ff82d7012356dc26898b97559f082e"}, - {file = "mkdocs_material-8.5.11.tar.gz", hash = "sha256:b0ea0513fd8cab323e8a825d6692ea07fa83e917bb5db042e523afecc7064ab7"}, -] -mkdocs-material-extensions = [ - {file = "mkdocs_material_extensions-1.1.1-py3-none-any.whl", hash = "sha256:e41d9f38e4798b6617ad98ca8f7f1157b1e4385ac1459ca1e4ea219b556df945"}, - {file = "mkdocs_material_extensions-1.1.1.tar.gz", hash = "sha256:9c003da71e2cc2493d910237448c672e00cefc800d3d6ae93d2fc69979e3bd93"}, -] -mypy = [ - {file = "mypy-0.990-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:aaf1be63e0207d7d17be942dcf9a6b641745581fe6c64df9a38deb562a7dbafa"}, - {file = "mypy-0.990-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d555aa7f44cecb7ea3c0ac69d58b1a5afb92caa017285a8e9c4efbf0518b61b4"}, - {file = "mypy-0.990-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f694d6d09a460b117dccb6857dda269188e3437c880d7b60fa0014fa872d1e9"}, - {file = "mypy-0.990-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:269f0dfb6463b8780333310ff4b5134425157ef0d2b1d614015adaf6d6a7eabd"}, - {file = "mypy-0.990-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8798c8ed83aa809f053abff08664bdca056038f5a02af3660de00b7290b64c47"}, - {file = "mypy-0.990-cp310-cp310-win_amd64.whl", hash = "sha256:47a9955214615108c3480a500cfda8513a0b1cd3c09a1ed42764ca0dd7b931dd"}, - {file = "mypy-0.990-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4a8a6c10f4c63fbf6ad6c03eba22c9331b3946a4cec97f008e9ffb4d3b31e8e2"}, - {file = "mypy-0.990-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cd2dd3730ba894ec2a2082cc703fbf3e95a08479f7be84912e3131fc68809d46"}, - {file = "mypy-0.990-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7da0005e47975287a92b43276e460ac1831af3d23032c34e67d003388a0ce8d0"}, - {file = "mypy-0.990-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:262c543ef24deb10470a3c1c254bb986714e2b6b1a67d66daf836a548a9f316c"}, - {file = "mypy-0.990-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3ff201a0c6d3ea029d73b1648943387d75aa052491365b101f6edd5570d018ea"}, - {file = "mypy-0.990-cp311-cp311-win_amd64.whl", hash = "sha256:1767830da2d1afa4e62b684647af0ff79b401f004d7fa08bc5b0ce2d45bcd5ec"}, - {file = "mypy-0.990-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6826d9c4d85bbf6d68cb279b561de6a4d8d778ca8e9ab2d00ee768ab501a9852"}, - {file = "mypy-0.990-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46897755f944176fbc504178422a5a2875bbf3f7436727374724842c0987b5af"}, - {file = "mypy-0.990-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0680389c34284287fe00e82fc8bccdea9aff318f7e7d55b90d967a13a9606013"}, - {file = "mypy-0.990-cp37-cp37m-win_amd64.whl", hash = "sha256:b08541a06eed35b543ae1a6b301590eb61826a1eb099417676ddc5a42aa151c5"}, - {file = "mypy-0.990-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:be88d665e76b452c26fb2bdc3d54555c01226fba062b004ede780b190a50f9db"}, - {file = "mypy-0.990-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b8f4a8213b1fd4b751e26b59ae0e0c12896568d7e805861035c7a15ed6dc9eb"}, - {file = "mypy-0.990-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b6f85c2ad378e3224e017904a051b26660087b3b76490d533b7344f1546d3ff"}, - {file = "mypy-0.990-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ee5f99817ee70254e7eb5cf97c1b11dda29c6893d846c8b07bce449184e9466"}, - {file = "mypy-0.990-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49082382f571c3186ce9ea0bd627cb1345d4da8d44a8377870f4442401f0a706"}, - {file = "mypy-0.990-cp38-cp38-win_amd64.whl", hash = "sha256:aba38e3dd66bdbafbbfe9c6e79637841928ea4c79b32e334099463c17b0d90ef"}, - {file = "mypy-0.990-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9d851c09b981a65d9d283a8ccb5b1d0b698e580493416a10942ef1a04b19fd37"}, - {file = "mypy-0.990-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d847dd23540e2912d9667602271e5ebf25e5788e7da46da5ffd98e7872616e8e"}, - {file = "mypy-0.990-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cc6019808580565040cd2a561b593d7c3c646badd7e580e07d875eb1bf35c695"}, - {file = "mypy-0.990-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a3150d409609a775c8cb65dbe305c4edd7fe576c22ea79d77d1454acd9aeda8"}, - {file = "mypy-0.990-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3227f14fe943524f5794679156488f18bf8d34bfecd4623cf76bc55958d229c5"}, - {file = "mypy-0.990-cp39-cp39-win_amd64.whl", hash = "sha256:c76c769c46a1e6062a84837badcb2a7b0cdb153d68601a61f60739c37d41cc74"}, - {file = "mypy-0.990-py3-none-any.whl", hash = "sha256:8f1940325a8ed460ba03d19ab83742260fa9534804c317224e5d4e5aa588e2d6"}, - {file = "mypy-0.990.tar.gz", hash = "sha256:72382cb609142dba3f04140d016c94b4092bc7b4d98ca718740dc989e5271b8d"}, -] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] -nodeenv = [ - {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, - {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, -] -packaging = [ - {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, - {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, -] -pathspec = [ - {file = "pathspec-0.10.3-py3-none-any.whl", hash = "sha256:3c95343af8b756205e2aba76e843ba9520a24dd84f68c22b9f93251507509dd6"}, - {file = "pathspec-0.10.3.tar.gz", hash = "sha256:56200de4077d9d0791465aa9095a01d421861e405b5096955051deefd697d6f6"}, -] -platformdirs = [ - {file = "platformdirs-2.6.0-py3-none-any.whl", hash = "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca"}, - {file = "platformdirs-2.6.0.tar.gz", hash = "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] -pycodestyle = [ - {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, - {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, -] -pydantic = [ - {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, - {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, - {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, - {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, - {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, - {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, - {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, - {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, - {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, - {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, -] -pyflakes = [ - {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, - {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, -] -pygments = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] -pyhumps = [ - {file = "pyhumps-3.8.0-py3-none-any.whl", hash = "sha256:060e1954d9069f428232a1adda165db0b9d8dfdce1d265d36df7fbff540acfd6"}, - {file = "pyhumps-3.8.0.tar.gz", hash = "sha256:498026258f7ee1a8e447c2e28526c0bea9407f9a59c03260aee4bd6c04d681a3"}, -] -pymdown-extensions = [ - {file = "pymdown_extensions-9.9-py3-none-any.whl", hash = "sha256:ac698c15265680db5eb13cd4342abfcde2079ac01e5486028f47a1b41547b859"}, - {file = "pymdown_extensions-9.9.tar.gz", hash = "sha256:0f8fb7b74a37a61cc34e90b2c91865458b713ec774894ffad64353a5fce85cfc"}, -] -pyrsistent = [ - {file = "pyrsistent-0.19.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a"}, - {file = "pyrsistent-0.19.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a"}, - {file = "pyrsistent-0.19.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:055ab45d5911d7cae397dc418808d8802fb95262751872c841c170b0dbf51eed"}, - {file = "pyrsistent-0.19.2-cp310-cp310-win32.whl", hash = "sha256:456cb30ca8bff00596519f2c53e42c245c09e1a4543945703acd4312949bfd41"}, - {file = "pyrsistent-0.19.2-cp310-cp310-win_amd64.whl", hash = "sha256:b39725209e06759217d1ac5fcdb510e98670af9e37223985f330b611f62e7425"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aede922a488861de0ad00c7630a6e2d57e8023e4be72d9d7147a9fcd2d30712"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:879b4c2f4d41585c42df4d7654ddffff1239dc4065bc88b745f0341828b83e78"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c43bec251bbd10e3cb58ced80609c5c1eb238da9ca78b964aea410fb820d00d6"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-win32.whl", hash = "sha256:d690b18ac4b3e3cab73b0b7aa7dbe65978a172ff94970ff98d82f2031f8971c2"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-win_amd64.whl", hash = "sha256:3ba4134a3ff0fc7ad225b6b457d1309f4698108fb6b35532d015dca8f5abed73"}, - {file = "pyrsistent-0.19.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a178209e2df710e3f142cbd05313ba0c5ebed0a55d78d9945ac7a4e09d923308"}, - {file = "pyrsistent-0.19.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e371b844cec09d8dc424d940e54bba8f67a03ebea20ff7b7b0d56f526c71d584"}, - {file = "pyrsistent-0.19.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111156137b2e71f3a9936baf27cb322e8024dac3dc54ec7fb9f0bcf3249e68bb"}, - {file = "pyrsistent-0.19.2-cp38-cp38-win32.whl", hash = "sha256:e5d8f84d81e3729c3b506657dddfe46e8ba9c330bf1858ee33108f8bb2adb38a"}, - {file = "pyrsistent-0.19.2-cp38-cp38-win_amd64.whl", hash = "sha256:9cd3e9978d12b5d99cbdc727a3022da0430ad007dacf33d0bf554b96427f33ab"}, - {file = "pyrsistent-0.19.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f1258f4e6c42ad0b20f9cfcc3ada5bd6b83374516cd01c0960e3cb75fdca6770"}, - {file = "pyrsistent-0.19.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21455e2b16000440e896ab99e8304617151981ed40c29e9507ef1c2e4314ee95"}, - {file = "pyrsistent-0.19.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfd880614c6237243ff53a0539f1cb26987a6dc8ac6e66e0c5a40617296a045e"}, - {file = "pyrsistent-0.19.2-cp39-cp39-win32.whl", hash = "sha256:71d332b0320642b3261e9fee47ab9e65872c2bd90260e5d225dabeed93cbd42b"}, - {file = "pyrsistent-0.19.2-cp39-cp39-win_amd64.whl", hash = "sha256:dec3eac7549869365fe263831f576c8457f6c833937c68542d08fde73457d291"}, - {file = "pyrsistent-0.19.2-py3-none-any.whl", hash = "sha256:ea6b79a02a28550c98b6ca9c35b9f492beaa54d7c5c9e9949555893c8a9234d0"}, - {file = "pyrsistent-0.19.2.tar.gz", hash = "sha256:bfa0351be89c9fcbcb8c9879b826f4353be10f58f8a677efab0c017bf7137ec2"}, -] -pytest = [ - {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, - {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, -] -pytest-mock = [ - {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, - {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -python-dotenv = [ - {file = "python-dotenv-0.21.0.tar.gz", hash = "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"}, - {file = "python_dotenv-0.21.0-py3-none-any.whl", hash = "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5"}, -] -python-schema-registry-client = [ - {file = "python-schema-registry-client-2.4.1.tar.gz", hash = "sha256:1e0562e3c6447a836f09b116cfe4d8904dee4e9d9d17adcfdf3e8916b897de86"}, -] -pyyaml = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] -pyyaml-env-tag = [ - {file = "pyyaml_env_tag-0.1-py3-none-any.whl", hash = "sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069"}, - {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, -] -requests = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, -] -responses = [ - {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, - {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, -] -rfc3986 = [ - {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, - {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, -] -rich = [ - {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, - {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, -] -setuptools = [ - {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, - {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, -] -shellingham = [ - {file = "shellingham-1.5.0-py2.py3-none-any.whl", hash = "sha256:a8f02ba61b69baaa13facdba62908ca8690a94b8119b69f5ec5873ea85f7391b"}, - {file = "shellingham-1.5.0.tar.gz", hash = "sha256:72fb7f5c63103ca2cb91b23dee0c71fe8ad6fbfd46418ef17dbe40db51592dad"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -snapshottest = [ - {file = "snapshottest-0.6.0-py2.py3-none-any.whl", hash = "sha256:9b177cffe0870c589df8ddbee0a770149c5474b251955bdbde58b7f32a4ec429"}, - {file = "snapshottest-0.6.0.tar.gz", hash = "sha256:bbcaf81d92d8e330042e5c928e13d9f035e99e91b314fe55fda949c2f17b653c"}, -] -sniffio = [ - {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, - {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, -] -soupsieve = [ - {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"}, - {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, -] -termcolor = [ - {file = "termcolor-2.1.1-py3-none-any.whl", hash = "sha256:fa852e957f97252205e105dd55bbc23b419a70fec0085708fc0515e399f304fd"}, - {file = "termcolor-2.1.1.tar.gz", hash = "sha256:67cee2009adc6449c650f6bcf3bdeed00c8ba53a8cda5362733c53e0a39fb70b"}, -] -toml = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -typer = [ - {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, - {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, -] -types-toml = [ - {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, - {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, -] -typing-extensions = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, -] -urllib3 = [ - {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, - {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, -] -verspec = [ - {file = "verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31"}, - {file = "verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e"}, -] -virtualenv = [ - {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, - {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, -] -wasmer = [ - {file = "wasmer-1.1.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:c2af4b907ae2dabcac41e316e811d5937c93adf1f8b05c5d49427f8ce0f37630"}, - {file = "wasmer-1.1.0-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:ab1ae980021e5ec0bf0c6cdd3b979b1d15a5f3eb2b8a32da8dcb1156e4a1e484"}, - {file = "wasmer-1.1.0-cp310-none-win_amd64.whl", hash = "sha256:d0d93aec6215893d33e803ef0a8d37bf948c585dd80ba0e23a83fafee820bc03"}, - {file = "wasmer-1.1.0-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:1e63d16bd6e2e2272d8721647831de5c537e0bb08002ee6d7abf167ec02d5178"}, - {file = "wasmer-1.1.0-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:85e6a5bf44853e8e6a12e947ee3412da9e84f7ce49fc165ba5dbd293e9c5c405"}, - {file = "wasmer-1.1.0-cp37-none-win_amd64.whl", hash = "sha256:a182a6eca9b46d895b4985fc822fab8da3d2f84fab74ca27e55a7430a7fcf336"}, - {file = "wasmer-1.1.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:214d9a3cfb577ea9449eb2b5f13adceae34c55365e4c3d930066beb86a7f67bc"}, - {file = "wasmer-1.1.0-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:b9e5605552bd7d2bc6337519b176defe83bc69b98abf3caaaefa4f7ec231d18a"}, - {file = "wasmer-1.1.0-cp38-none-win_amd64.whl", hash = "sha256:20b5190112e2e94a8947967f2bc683c9685855d0f34130d8434c87a55216a3bd"}, - {file = "wasmer-1.1.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:ee442f0970f40ec5e32011c92fd753fb2061da0faa13de13fafc730c31be34e3"}, - {file = "wasmer-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa112198b743cff2e391230436813fb4b244a24443e37866522b7197e3a034da"}, - {file = "wasmer-1.1.0-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:c0b37117f6d3ff51ee96431c7d224d99799b08d174e30fcd0fcd7e2e3cb8140c"}, - {file = "wasmer-1.1.0-cp39-none-win_amd64.whl", hash = "sha256:a0a4730ec4907a4cb0d9d4a77ea2608c2c814f22a22b73fc80be0f110e014836"}, - {file = "wasmer-1.1.0-py3-none-any.whl", hash = "sha256:2caf8c67feae9cd4246421551036917811c446da4f27ad4c989521ef42751931"}, -] -wasmer-compiler-cranelift = [ - {file = "wasmer_compiler_cranelift-1.1.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:9869910179f39696a020edc5689f7759257ac1cce569a7a0fcf340c59788baad"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:405546ee864ac158a4107f374dfbb1c8d6cfb189829bdcd13050143a4bd98f28"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp310-none-win_amd64.whl", hash = "sha256:bdf75af9ef082e6aeb752550f694273340ece970b65099e0746db0f972760d11"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:7d9c782b7721789b16e303b7e70c59df370896dd62b77e2779e3a44b4e1aa20c"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:ff7dd5bd69030b63521c24583bf0f5457cd2580237340b91ce35370f72a4a1cc"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp37-none-win_amd64.whl", hash = "sha256:447285402e366a34667a674db70458c491acd6940b797c175c0b0027f48e64bb"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:55a524985179f6b7b88ac973e8fac5a2574d3b125a966fba75fedd5a2525e484"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:bd03db5a916ead51b442c66acad38847dfe127cf90b2019b1680f1920c4f8d06"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp38-none-win_amd64.whl", hash = "sha256:157d87cbd1d04adbad55b50cb4bedc28e444caf74797fd96df17390667e58699"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:ff25fc99ebafa04a6c271d08a90d17b927930e3019a2b333c7cfb48ba32c6f71"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9697ae082317a56776df8ff7df8c922eac38488ef38d3478fe5f0ca144c185ab"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:2a4349b1ddd727bd46bc5ede741839dcfc5153c52f064a83998c4150d5d4a85c"}, - {file = "wasmer_compiler_cranelift-1.1.0-cp39-none-win_amd64.whl", hash = "sha256:32fe38614fccc933da77ee4372904a5fa9c12b859209a2e4048a8284c4c371f2"}, - {file = "wasmer_compiler_cranelift-1.1.0-py3-none-any.whl", hash = "sha256:200fea80609cfb088457327acf66d5aa61f4c4f66b5a71133ada960b534c7355"}, -] -watchdog = [ +files = [ {file = "watchdog-2.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ed91c3ccfc23398e7aa9715abf679d5c163394b8cad994f34f156d57a7c163dc"}, {file = "watchdog-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:76a2743402b794629a955d96ea2e240bd0e903aa26e02e93cd2d57b33900962b"}, {file = "watchdog-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:920a4bda7daa47545c3201a3292e99300ba81ca26b7569575bd086c865889090"}, @@ -1573,3 +1565,11 @@ watchdog = [ {file = "watchdog-2.2.0-py3-none-win_ia64.whl", hash = "sha256:ad0150536469fa4b693531e497ffe220d5b6cd76ad2eda474a5e641ee204bbb6"}, {file = "watchdog-2.2.0.tar.gz", hash = "sha256:83cf8bc60d9c613b66a4c018051873d6273d9e45d040eed06d6a96241bd8ec01"}, ] + +[package.extras] +watchmedo = ["PyYAML (>=3.10)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "176f95fcc62a48d196060f20a6e99ba8d212887a2b8b376c64bae8aa8276229e" diff --git a/pyproject.toml b/pyproject.toml index 18b5b81bb..53567da09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ pre-commit = "^2.19.0" mypy = "^0.990" flake8 = "^4.0.1" black = "^22.3.0" -isort = "^5.10.1" +isort = "^5.12.0" [tool.poetry.group.docs] optional = true From 30d47c296df37a75883cc9f48a1c55501476faff Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 30 Jan 2023 10:13:19 +0100 Subject: [PATCH 40/44] Fix isort bug https://github.com/PyCQA/isort/issues/2079 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bb667112a..e88a287e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort args: ["--settings", "setup.cfg"] From f0927dbc238a06e088a29be743e18faa41565f11 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 30 Jan 2023 10:18:25 +0100 Subject: [PATCH 41/44] Update Poetry --- .github/actions/update-docs/action.yml | 1 + .github/workflows/ci.yaml | 13 +++++++------ .github/workflows/release.yaml | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/actions/update-docs/action.yml b/.github/actions/update-docs/action.yml index 1a3ca0c14..3391ea7b7 100644 --- a/.github/actions/update-docs/action.yml +++ b/.github/actions/update-docs/action.yml @@ -24,6 +24,7 @@ runs: - name: Install python and set up Poetry uses: bakdata/ci-templates/actions/python-setup-poetry@v1.5.2 with: + poetry-version: "1.3.2" python-version: "3.10" - name: Update documentation branch with mike diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 51b6deb4a..ed553d733 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,11 +1,11 @@ name: CI on: - pull_request: - types: [opened, ready_for_review, synchronize] - push: - branches: - - main + pull_request: + types: [opened, ready_for_review, synchronize] + push: + branches: + - main jobs: test: @@ -20,8 +20,9 @@ jobs: - uses: actions/checkout@v3 - name: Install python and set up Poetry - uses: bakdata/ci-templates/actions/python-setup-poetry@v1.5.2 + uses: bakdata/ci-templates/actions/python-setup-poetry@v1.5.3 with: + poetry-version: "1.3.2" python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2dd3d14ea..5927d2872 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,7 +20,7 @@ jobs: release-type: ${{ github.event.inputs.release-type }} publish-to-test: false python-version: "3.10" - poetry-version: "1.2.2" + poetry-version: "1.3.2" secrets: github-email: ${{ secrets.GH_EMAIL }} github-username: ${{ secrets.GH_USERNAME }} From e4842dc7ed961f3be34684fe0c7e2eaaeb3de035 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 30 Jan 2023 10:30:09 +0100 Subject: [PATCH 42/44] Fix Poetry publish to TestPyPI --- .github/workflows/publish-dev.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-dev.yaml b/.github/workflows/publish-dev.yaml index 779e00e46..00456251d 100644 --- a/.github/workflows/publish-dev.yaml +++ b/.github/workflows/publish-dev.yaml @@ -19,7 +19,7 @@ jobs: - name: Bump version id: bump_version - uses: bakdata/ci-templates/actions/python-poetry-bump-version@v1.5.2 + uses: bakdata/ci-templates/actions/python-poetry-bump-version@v1.5.3 with: release-type: "patch" @@ -40,7 +40,9 @@ jobs: shell: bash - name: Release to TestPyPI - uses: bakdata/ci-templates/actions/python-poetry-release@v1.5.2 + uses: bakdata/ci-templates/actions/python-poetry-release@v1.5.3 with: + python-version: "3.10" + poetry-version: "1.3.2" publish-to-test: true pypi-token: ${{ secrets.test-pypi-token }} From 4e78163cf40600def17e3bd65b53ad6ea1704d24 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 30 Jan 2023 10:34:53 +0100 Subject: [PATCH 43/44] Add `KubernetesApp` & `KafkaApp` to schema --- gen_schema.py | 11 +- kpops/components/base_components/kafka_app.py | 7 +- .../base_components/kubernetes_app.py | 7 +- .../base_components/models/to_section.py | 3 +- .../producer/producer_app.py | 4 +- .../streams_bootstrap/streams/streams_app.py | 4 +- schema.json | 400 ++++++++++++------ 7 files changed, 302 insertions(+), 134 deletions(-) diff --git a/gen_schema.py b/gen_schema.py index 621ba62ea..0ffd68b7a 100644 --- a/gen_schema.py +++ b/gen_schema.py @@ -4,10 +4,12 @@ from pydantic.fields import ModelField from pydantic.schema import SkipField +from kpops.components.base_components.kafka_app import KafkaApp from kpops.components.base_components.kafka_connect import ( KafkaSinkConnector, KafkaSourceConnector, ) +from kpops.components.base_components.kubernetes_app import KubernetesApp from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp @@ -24,7 +26,14 @@ def field_schema(field: ModelField, **kwargs: Any) -> Any: schema.field_schema = field_schema -PipelineComponent = StreamsApp | ProducerApp | KafkaSourceConnector | KafkaSinkConnector +PipelineComponent = ( + KubernetesApp + | KafkaApp + | StreamsApp + | ProducerApp + | KafkaSourceConnector + | KafkaSinkConnector +) AnnotatedPipelineComponent = Annotated[ diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 15859d1fd..bf3202bc7 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -1,9 +1,9 @@ from __future__ import annotations import logging -from typing import ClassVar +from typing import ClassVar, Literal -from pydantic import BaseModel, Extra +from pydantic import BaseModel, Extra, Field from typing_extensions import override from kpops.component_handlers.helm_wrapper.helm import Helm @@ -42,6 +42,9 @@ class KafkaApp(KubernetesApp): """ type: ClassVar[str] = "kafka-app" + schema_type: Literal["kafka-app"] = Field( # type: ignore[assignment] + default="kafka-app", exclude=True + ) app: KafkaAppConfig repo_config: HelmRepoConfig = HelmRepoConfig( repository_name="bakdata-streams-bootstrap", diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index e545e9e31..ca115c24f 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -3,9 +3,9 @@ import logging import re from functools import cached_property -from typing import ClassVar +from typing import ClassVar, Literal -from pydantic import BaseModel, Extra +from pydantic import BaseModel, Extra, Field from typing_extensions import override from kpops.component_handlers.helm_wrapper.helm import Helm @@ -35,6 +35,9 @@ class KubernetesApp(PipelineComponent): """Base Kubernetes app""" type: ClassVar[str] = "kubernetes-app" + schema_type: Literal["kubernetes-app"] = Field( # type: ignore[assignment] + default="kubernetes-app", exclude=True + ) app: KubernetesAppConfig repo_config: HelmRepoConfig | None = None namespace: str diff --git a/kpops/components/base_components/models/to_section.py b/kpops/components/base_components/models/to_section.py index c3be1caa5..c6f8ffe61 100644 --- a/kpops/components/base_components/models/to_section.py +++ b/kpops/components/base_components/models/to_section.py @@ -5,8 +5,7 @@ class OutputTopicTypes(str, Enum): - """ - Types of output topic + """Types of output topic. error (error topic), output (output topic), and extra topics. Every extra topic must have a role. """ diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 85be72ca4..a2a7f8117 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -22,7 +22,9 @@ class ProducerApp(KafkaApp): """ type: ClassVar[str] = "producer" - schema_type: Literal["producer"] = Field(default="producer", exclude=True) + schema_type: Literal["producer"] = Field( # type: ignore[assignment] + default="producer", exclude=True + ) app: ProducerValues class Config(BaseConfig): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index b133555f3..798081277 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -16,7 +16,9 @@ class StreamsApp(KafkaApp): """ type: ClassVar[str] = "streams-app" - schema_type: Literal["streams-app"] = Field(default="streams-app", exclude=True) + schema_type: Literal["streams-app"] = Field( # type: ignore[assignment] + default="streams-app", exclude=True + ) app: StreamsAppConfig class Config(BaseConfig): diff --git a/schema.json b/schema.json index eee48c00d..3a32a5601 100644 --- a/schema.json +++ b/schema.json @@ -5,6 +5,8 @@ "discriminator": { "propertyName": "type", "mapping": { + "kubernetes-app": "#/definitions/KubernetesApp", + "kafka-app": "#/definitions/KafkaApp", "streams-app": "#/definitions/StreamsApp", "producer": "#/definitions/ProducerApp", "kafka-source-connector": "#/definitions/KafkaSourceConnector", @@ -12,6 +14,12 @@ } }, "oneOf": [ + { + "$ref": "#/definitions/KubernetesApp" + }, + { + "$ref": "#/definitions/KafkaApp" + }, { "$ref": "#/definitions/StreamsApp" }, @@ -72,123 +80,14 @@ ], "additionalProperties": false }, - "StreamsConfig": { - "title": "StreamsConfig", - "description": "Streams Bootstrap streams section", - "type": "object", - "properties": { - "brokers": { - "title": "Brokers", - "type": "string" - }, - "schemaRegistryUrl": { - "title": "Schemaregistryurl", - "type": "string" - }, - "inputTopics": { - "title": "Inputtopics", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "inputPattern": { - "title": "Inputpattern", - "type": "string" - }, - "extraInputTopics": { - "title": "Extrainputtopics", - "default": {}, - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "extraInputPatterns": { - "title": "Extrainputpatterns", - "default": {}, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "extraOutputTopics": { - "title": "Extraoutputtopics", - "default": {}, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "outputTopic": { - "title": "Outputtopic", - "type": "string" - }, - "errorTopic": { - "title": "Errortopic", - "type": "string" - }, - "config": { - "title": "Config", - "default": {}, - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "required": [ - "brokers" - ] - }, - "StreamsAppAutoScaling": { - "title": "StreamsAppAutoScaling", + "KubernetesAppConfig": { + "title": "KubernetesAppConfig", "type": "object", - "properties": { - "consumergroup": { - "title": "Consumergroup", - "type": "string" - }, - "topics": { - "title": "Topics", - "default": [], - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "consumergroup" - ] - }, - "StreamsAppConfig": { - "title": "StreamsAppConfig", - "description": "StreamsBoostrap app configurations.\n\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", - "type": "object", - "properties": { - "streams": { - "$ref": "#/definitions/StreamsConfig" - }, - "nameOverride": { - "title": "Nameoverride", - "type": "string" - }, - "autoscaling": { - "$ref": "#/definitions/StreamsAppAutoScaling" - } - }, - "required": [ - "streams" - ] + "properties": {} }, "OutputTopicTypes": { "title": "OutputTopicTypes", - "description": "Types of output topic\nerror (error topic), output (output topic), and extra topics. Every extra topic must have a role.", + "description": "Types of output topic.\n error (error topic), output (output topic), and extra topics. Every extra topic must have a role.\n ", "enum": [ "error", "output", @@ -315,9 +214,9 @@ "url" ] }, - "StreamsApp": { - "title": "StreamsApp", - "description": "StreamsApp component that configures a streams bootstrap app", + "KubernetesApp": { + "title": "KubernetesApp", + "description": "Base Kubernetes app", "type": "object", "properties": { "name": { @@ -328,11 +227,96 @@ "$ref": "#/definitions/FromSection" }, "app": { - "$ref": "#/definitions/StreamsAppConfig" + "$ref": "#/definitions/KubernetesAppConfig" }, "to": { "$ref": "#/definitions/ToSection" }, + "type": { + "title": "Schema Type", + "default": "kubernetes-app", + "enum": [ + "kubernetes-app" + ], + "type": "string" + }, + "repoConfig": { + "$ref": "#/definitions/HelmRepoConfig" + }, + "namespace": { + "title": "Namespace", + "type": "string" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "app", + "namespace" + ] + }, + "KafkaStreamsConfig": { + "title": "KafkaStreamsConfig", + "type": "object", + "properties": { + "brokers": { + "title": "Brokers", + "type": "string" + }, + "schemaRegistryUrl": { + "title": "Schemaregistryurl", + "type": "string" + } + }, + "required": [ + "brokers" + ] + }, + "KafkaAppConfig": { + "title": "KafkaAppConfig", + "type": "object", + "properties": { + "streams": { + "$ref": "#/definitions/KafkaStreamsConfig" + }, + "nameOverride": { + "title": "Nameoverride", + "type": "string" + } + }, + "required": [ + "streams" + ] + }, + "KafkaApp": { + "title": "KafkaApp", + "description": "Base component for Kafka-based components.\nProducer or streaming apps should inherit from this class.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "from": { + "$ref": "#/definitions/FromSection" + }, + "app": { + "$ref": "#/definitions/KafkaAppConfig" + }, + "to": { + "$ref": "#/definitions/ToSection" + }, + "type": { + "title": "Schema Type", + "default": "kafka-app", + "enum": [ + "kafka-app" + ], + "type": "string" + }, "repoConfig": { "title": "Repoconfig", "default": { @@ -359,6 +343,145 @@ "title": "Version", "default": "2.7.0", "type": "string" + } + }, + "required": [ + "name", + "app", + "namespace" + ] + }, + "StreamsConfig": { + "title": "StreamsConfig", + "description": "Streams Bootstrap streams section", + "type": "object", + "properties": { + "brokers": { + "title": "Brokers", + "type": "string" + }, + "schemaRegistryUrl": { + "title": "Schemaregistryurl", + "type": "string" + }, + "inputTopics": { + "title": "Inputtopics", + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "inputPattern": { + "title": "Inputpattern", + "type": "string" + }, + "extraInputTopics": { + "title": "Extrainputtopics", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "extraInputPatterns": { + "title": "Extrainputpatterns", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "extraOutputTopics": { + "title": "Extraoutputtopics", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "outputTopic": { + "title": "Outputtopic", + "type": "string" + }, + "errorTopic": { + "title": "Errortopic", + "type": "string" + }, + "config": { + "title": "Config", + "default": {}, + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "brokers" + ] + }, + "StreamsAppAutoScaling": { + "title": "StreamsAppAutoScaling", + "type": "object", + "properties": { + "consumergroup": { + "title": "Consumergroup", + "type": "string" + }, + "topics": { + "title": "Topics", + "default": [], + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "consumergroup" + ] + }, + "StreamsAppConfig": { + "title": "StreamsAppConfig", + "description": "StreamsBoostrap app configurations.\n\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "type": "object", + "properties": { + "streams": { + "$ref": "#/definitions/StreamsConfig" + }, + "nameOverride": { + "title": "Nameoverride", + "type": "string" + }, + "autoscaling": { + "$ref": "#/definitions/StreamsAppAutoScaling" + } + }, + "required": [ + "streams" + ] + }, + "StreamsApp": { + "title": "StreamsApp", + "description": "StreamsApp component that configures a streams bootstrap app", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "from": { + "$ref": "#/definitions/FromSection" + }, + "app": { + "$ref": "#/definitions/StreamsAppConfig" + }, + "to": { + "$ref": "#/definitions/ToSection" }, "type": { "title": "Schema Type", @@ -367,6 +490,33 @@ "streams-app" ], "type": "string" + }, + "repoConfig": { + "title": "Repoconfig", + "default": { + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/", + "repo_auth_flags": { + "username": null, + "password": null, + "ca_file": null, + "insecure_skip_tls_verify": false + } + }, + "allOf": [ + { + "$ref": "#/definitions/HelmRepoConfig" + } + ] + }, + "namespace": { + "title": "Namespace", + "type": "string" + }, + "version": { + "title": "Version", + "default": "2.7.0", + "type": "string" } }, "required": [ @@ -438,6 +588,14 @@ "to": { "$ref": "#/definitions/ToSection" }, + "type": { + "title": "Schema Type", + "default": "producer", + "enum": [ + "producer" + ], + "type": "string" + }, "repoConfig": { "title": "Repoconfig", "default": { @@ -464,14 +622,6 @@ "title": "Version", "default": "2.7.0", "type": "string" - }, - "type": { - "title": "Schema Type", - "default": "producer", - "enum": [ - "producer" - ], - "type": "string" } }, "required": [ From 0c8d2f215c67c0d75174e4d66222b1b823b9e914 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 30 Jan 2023 11:09:50 +0100 Subject: [PATCH 44/44] Generate schema for config.yaml --- gen_schema.py | 13 +- schema_config.json | 229 ++++++++++++++++++++++++++++ schema.json => schema_pipeline.json | 0 3 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 schema_config.json rename schema.json => schema_pipeline.json (100%) diff --git a/gen_schema.py b/gen_schema.py index 0ffd68b7a..b90e6f7b4 100644 --- a/gen_schema.py +++ b/gen_schema.py @@ -1,9 +1,11 @@ +from pathlib import Path from typing import Annotated, Any, Sequence from pydantic import Field, schema, schema_json_of from pydantic.fields import ModelField from pydantic.schema import SkipField +from kpops.cli.pipeline_config import PipelineConfig from kpops.components.base_components.kafka_app import KafkaApp from kpops.components.base_components.kafka_connect import ( KafkaSinkConnector, @@ -13,6 +15,12 @@ from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp + +def write(contents: str, path: Path) -> None: + with open(path, "w") as f: + print(contents, file=f) + + original_field_schema = schema.field_schema @@ -40,11 +48,14 @@ def field_schema(field: ModelField, **kwargs: Any) -> Any: PipelineComponent, Field(discriminator="schema_type") ] + schema = schema_json_of( Sequence[AnnotatedPipelineComponent], title="kpops pipeline schema", by_alias=True, indent=4, ).replace("schema_type", "type") +write(schema, Path("schema_pipeline.json")) -print(schema) +schema = schema_json_of(PipelineConfig, title="kpops config schema", indent=4) +write(schema, Path("schema_config.json")) diff --git a/schema_config.json b/schema_config.json new file mode 100644 index 000000000..074c513c9 --- /dev/null +++ b/schema_config.json @@ -0,0 +1,229 @@ +{ + "title": "kpops config schema", + "$ref": "#/definitions/PipelineConfig", + "definitions": { + "TopicNameConfig": { + "title": "TopicNameConfig", + "description": "Base class for settings, allowing values to be overridden by environment variables.\n\nThis is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),\nHeroku and any 12 factor app design.", + "type": "object", + "properties": { + "default_output_topic_name": { + "title": "Default Output Topic Name", + "description": "Configures the value for the variable ${output_topic_name}", + "default": "${pipeline_name}-${component_name}", + "env_names": [ + "default_output_topic_name" + ], + "type": "string" + }, + "default_error_topic_name": { + "title": "Default Error Topic Name", + "description": "Configures the value for the variable ${error_topic_name}", + "default": "${pipeline_name}-${component_name}-error", + "env_names": [ + "default_error_topic_name" + ], + "type": "string" + } + }, + "additionalProperties": false + }, + "HelmConfig": { + "title": "HelmConfig", + "type": "object", + "properties": { + "context": { + "title": "Context", + "env_names": [ + "context" + ], + "type": "string" + }, + "debug": { + "title": "Debug", + "default": false, + "env_names": [ + "debug" + ], + "type": "boolean" + } + }, + "additionalProperties": false + }, + "HelmDiffConfig": { + "title": "HelmDiffConfig", + "type": "object", + "properties": { + "enable": { + "title": "Enable", + "default": false, + "type": "boolean" + }, + "ignore": { + "title": "Ignore", + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + } + }, + "PipelineConfig": { + "title": "PipelineConfig", + "description": "Base class for settings, allowing values to be overridden by environment variables.\n\nThis is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),\nHeroku and any 12 factor app design.", + "type": "object", + "properties": { + "defaults_path": { + "title": "Defaults Path", + "description": "The path to the folder containing the defaults file and the environment defaults files.", + "env_names": [ + "defaults_path" + ], + "type": "string", + "format": "path" + }, + "environment": { + "title": "Environment", + "description": "The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).", + "env": "KPOPS_ENVIRONMENT", + "example": "development", + "env_names": [ + "kpops_environment" + ], + "type": "string" + }, + "broker": { + "title": "Broker", + "description": "The kafka broker address.", + "env": "KPOPS_KAFKA_BROKER", + "env_names": [ + "kpops_kafka_broker" + ], + "type": "string" + }, + "defaults_filename_prefix": { + "title": "Defaults Filename Prefix", + "description": "The name of the defaults file and the prefix of the defaults environment file.", + "default": "defaults", + "env_names": [ + "defaults_filename_prefix" + ], + "type": "string" + }, + "topic_name_config": { + "title": "Topic Name Config", + "description": "Configure the topic name variables you can use in the pipeline definition.", + "default": { + "default_output_topic_name": "${pipeline_name}-${component_name}", + "default_error_topic_name": "${pipeline_name}-${component_name}-error" + }, + "env_names": [ + "topic_name_config" + ], + "allOf": [ + { + "$ref": "#/definitions/TopicNameConfig" + } + ] + }, + "kafka_rest_host": { + "title": "Kafka Rest Host", + "description": "Address to the rest proxy REST API.", + "env": "KPOPS_REST_PROXY_HOST", + "example": "http://localhost:8082", + "env_names": [ + "kpops_rest_proxy_host" + ], + "type": "string" + }, + "kafka_connect_host": { + "title": "Kafka Connect Host", + "description": "Address to the kafka connect REST API.", + "env": "KPOPS_CONNECT_HOST", + "example": "http://localhost:8083", + "env_names": [ + "kpops_connect_host" + ], + "type": "string" + }, + "timeout": { + "title": "Timeout", + "description": "The timeout in seconds that specifies when actions like deletion or deploy timeout.", + "default": 300, + "env": "KPOPS_TIMEOUT", + "env_names": [ + "kpops_timeout" + ], + "type": "integer" + }, + "pipeline_prefix": { + "title": "Pipeline Prefix", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "default": "${pipeline_name}-", + "env": "KPOPS_PIPELINE_PREFIX", + "env_names": [ + "kpops_pipeline_prefix" + ], + "type": "string" + }, + "helm_config": { + "title": "Helm Config", + "default": { + "context": null, + "debug": false + }, + "env_names": [ + "helm_config" + ], + "allOf": [ + { + "$ref": "#/definitions/HelmConfig" + } + ] + }, + "helm_diff_config": { + "title": "Helm Diff Config", + "default": { + "enable": false, + "ignore": [] + }, + "env_names": [ + "helm_diff_config" + ], + "allOf": [ + { + "$ref": "#/definitions/HelmDiffConfig" + } + ] + }, + "retain_clean_jobs": { + "title": "Retain Clean Jobs", + "description": "Whether to retain clean up jobs in the cluster or uninstall the, after completion.", + "default": false, + "env": "KPOPS_RETAIN_CLEAN_JOBS", + "env_names": [ + "kpops_retain_clean_jobs" + ], + "type": "boolean" + }, + "schema_registry_url": { + "title": "Schema Registry Url", + "description": "The URL to schema registry.", + "example": "http://localhost:8081", + "env": "KPOPS_SCHEMA_REGISTRY_URL", + "env_names": [ + "kpops_schema_registry_url" + ], + "type": "string" + } + }, + "required": [ + "defaults_path", + "environment", + "broker" + ], + "additionalProperties": false + } + } +} diff --git a/schema.json b/schema_pipeline.json similarity index 100% rename from schema.json rename to schema_pipeline.json