diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 836e89d78a8..8f2d6c315c0 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -59,6 +59,19 @@ helm install gateway bitnami/contour -n flyte | cloud_events.enable | bool | `false` | | | cloud_events.eventsPublisher.eventTypes[0] | string | `"all"` | | | cloud_events.eventsPublisher.topicName | string | `"arn:aws:sns:us-east-2:123456:123-my-topic"` | | +| cloud_events.kafka | object | `{"brokers":["mybroker:443"],"saslConfig":{"enabled":false,"handshake":true,"mechanism":"PLAIN","password":"","user":"kafka"},"tlsConfig":{"certPath":"/etc/ssl/certs/kafka-client.crt","enabled":false,"keyPath":"/etc/ssl/certs/kafka-client.key"},"version":"3.7.0"}` | Configuration for sending cloud events to Kafka | +| cloud_events.kafka.brokers | list | `["mybroker:443"]` | The kafka brokers to talk to | +| cloud_events.kafka.saslConfig | object | `{"enabled":false,"handshake":true,"mechanism":"PLAIN","password":"","user":"kafka"}` | SASL based authentication | +| cloud_events.kafka.saslConfig.enabled | bool | `false` | Whether to use SASL authentication | +| cloud_events.kafka.saslConfig.handshake | bool | `true` | Whether the send the SASL handsahke first | +| cloud_events.kafka.saslConfig.mechanism | string | `"PLAIN"` | Which SASL mechanism to use. Defaults to PLAIN | +| cloud_events.kafka.saslConfig.password | string | `""` | The password for the kafka user | +| cloud_events.kafka.saslConfig.user | string | `"kafka"` | The kafka user | +| cloud_events.kafka.tlsConfig | object | `{"certPath":"/etc/ssl/certs/kafka-client.crt","enabled":false,"keyPath":"/etc/ssl/certs/kafka-client.key"}` | Certificate based authentication | +| cloud_events.kafka.tlsConfig.certPath | string | `"/etc/ssl/certs/kafka-client.crt"` | Path to the client certificate | +| cloud_events.kafka.tlsConfig.enabled | bool | `false` | Whether to use certificate based authentication | +| cloud_events.kafka.tlsConfig.keyPath | string | `"/etc/ssl/certs/kafka-client.key"` | Path to the client private key | +| cloud_events.kafka.version | string | `"3.7.0"` | The version of Kafka | | cloud_events.secretName | string | `""` | The name of the secret to use to alternatively load in cloud events configuration via a secret. Useful when the configuration contains secrets. | | cloud_events.type | string | `"aws"` | | | cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"nodeSelector":{},"podAnnotations":{},"podEnv":{},"podLabels":{},"prometheus":{"enabled":false,"path":"/metrics","port":10254},"resources":{},"service_account_name":"flyteadmin","standaloneDeployment":false,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 986d6958275..d9469457ff3 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -956,6 +956,36 @@ cloud_events: topicName: "arn:aws:sns:us-east-2:123456:123-my-topic" eventTypes: - all # Or workflow, node, task. Or "*" + # -- Configuration for sending cloud events to Kafka + kafka: + # -- The version of Kafka + version: "3.7.0" + # -- The kafka brokers to talk to + brokers: + - mybroker:443 + # -- SASL based authentication + saslConfig: + # -- Whether to use SASL authentication + enabled: false + # -- The kafka user + user: kafka + # -- The password for the kafka user + password: "" + # -- Whether the send the SASL handsahke first + handshake: true + # -- Which SASL mechanism to use. Defaults to PLAIN + mechanism: PLAIN + # -- Certificate based authentication + tlsConfig: + # -- Whether to use certificate based authentication + enabled: false + # -- Path to the client certificate + certPath: /etc/ssl/certs/kafka-client.crt + # -- Path to the client private key + keyPath: /etc/ssl/certs/kafka-client.key + + + # -- Configuration for the Cluster resource manager component. This is an optional component, that enables automatic # cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index 73f19dfd44e..bcd1d4b50c9 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -816,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: ak5wVTFQVjRHMm5ZanVNUQ== + haSharedSecret: cThGYzV0YjVIQTc3QnpPTg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1413,7 +1413,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: db4b259a37cc362add2a4fd4c52954eabc67e69e2c399a292605415c70da4a2b + checksum/secret: 4c2776b79901c6d3e2e1b3fcc601f01debe089110c7f9253971e009368eef3f3 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index ce5fe830ddc..b1d876a5968 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -798,7 +798,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: RGZkSmNtV3k4dDZYd0pHVw== + haSharedSecret: QldNYlIzMFpBd3VoNzBVQw== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1362,7 +1362,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: d0a1f670be47a94b928141eae8a50733a775d074aee3a78db555b0728c90718e + checksum/secret: a7865be2c1fc4f96ad00701e8085b6f0504aae0abed1a9ff7513c09d6ca9fae2 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 419dc50d3a9..0a2582603b0 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: amliZ0l4QXczU0ZjUUloWQ== + haSharedSecret: ZGZiYmFVbTR1UVJMYkhnNw== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 4c93d218f3a1654f7eb3a238a4b1f57fbfda80cd8e5c4aaf5a9286a93f4a94f2 + checksum/secret: d50c3e69284bbb1ab6ec1175d26a89d95075afddea5755db0b7adebe7e601a27 labels: app: docker-registry release: flyte-sandbox diff --git a/flyteadmin/pkg/runtime/interfaces/application_configuration.go b/flyteadmin/pkg/runtime/interfaces/application_configuration.go index 809aadb9417..243b38b343d 100644 --- a/flyteadmin/pkg/runtime/interfaces/application_configuration.go +++ b/flyteadmin/pkg/runtime/interfaces/application_configuration.go @@ -12,7 +12,6 @@ import ( "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flytestdlib/config" "github.com/flyteorg/flyte/flytestdlib/database" - "github.com/flyteorg/flyte/flytestdlib/logger" ) // DbConfig is used to for initiating the database connection with the store that holds registered @@ -268,16 +267,15 @@ type KafkaConfig struct { // kafka broker addresses Brokers []string `json:"brokers"` // sasl config - SASLConfig SASLConfig `json:"sasl_config"` + SASLConfig SASLConfig `json:"saslConfig"` // tls config - TLSConfig TLSConfig `json:"tls_config"` + TLSConfig TLSConfig `json:"tlsConfig"` } func (k KafkaConfig) UpdateSaramaConfig(ctx context.Context, s *sarama.Config) { var err error s.Version, err = sarama.ParseKafkaVersion(k.Version) if err != nil { - logger.Fatalf(ctx, "failed to parse kafka version, %v", err) panic(err) } @@ -301,7 +299,6 @@ func (k KafkaConfig) UpdateSaramaConfig(ctx context.Context, s *sarama.Config) { if k.TLSConfig.KeyPath != "" && k.TLSConfig.CertPath != "" { cert, err := tls.LoadX509KeyPair(k.TLSConfig.CertPath, k.TLSConfig.KeyPath) if err != nil { - logger.Fatalf(ctx, "failed to load kafka client keypair: %v", err) panic(err) } s.Net.TLS.Config.Certificates = []tls.Certificate{cert}