From f434fd029eff3cc1784020c46c04431c6b768a0e Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Tue, 4 Aug 2020 17:55:14 -0700 Subject: [PATCH 1/8] docs: mixed types of events in single EventSource --- docs/assets/alpha.svg | 1 + docs/eventsource-deployment-strategies.md | 49 ++++++++++++++ docs/eventsource-names.md | 41 ++++++++++++ docs/eventsource-with-multiple-events.md | 78 +++++++++++++++++++++++ pkg/apis/common/common.go | 6 ++ 5 files changed, 175 insertions(+) create mode 100644 docs/assets/alpha.svg create mode 100644 docs/eventsource-deployment-strategies.md create mode 100644 docs/eventsource-names.md create mode 100644 docs/eventsource-with-multiple-events.md diff --git a/docs/assets/alpha.svg b/docs/assets/alpha.svg new file mode 100644 index 0000000000..471179bef4 --- /dev/null +++ b/docs/assets/alpha.svg @@ -0,0 +1 @@ + ALPHAALPHA \ No newline at end of file diff --git a/docs/eventsource-deployment-strategies.md b/docs/eventsource-deployment-strategies.md new file mode 100644 index 0000000000..6612cfb8d0 --- /dev/null +++ b/docs/eventsource-deployment-strategies.md @@ -0,0 +1,49 @@ +# EventSource Deployment Strategies + +EventSource controller creates a k8s deployment for each EventSource object to +watch the events. Some of the event source types do not allow multiple live +clients with same attributes (i.e. multiple clients with same `clientID` +connecting to a NATS server), or multiple event source PODs will generate +duplicated events to downstream, so the deployment strategy and replica numbers +are different for different event sources. + +## Rolling Update Strategy + +`Rolling Update` strategy is applied to the following EventSource types: + +- AMQP +- AWS SNS +- AWS SQS +- Github +- Gitlab +- NetApp Storage GRID +- Slack +- Stripe +- Webhook + +## Recreate Strategy + +`Recreate` strategy is applied to the following EventSource types: + +- Azure Events Hub +- Kafka +- GCP PubSub +- File +- HDFS +- NATS +- Minio +- MQTT +- Emitter +- NSQ +- Pulsar +- Redis +- Resource +- Calendar + +### Replicas + +The field `replica` for EventSource of these `Recreate` types are ignored, the +deployment is always created with `replica=1`. + +**Please DO NOT manually scale up the replicas, that will bring unexpected +behaviors!** diff --git a/docs/eventsource-names.md b/docs/eventsource-names.md new file mode 100644 index 0000000000..7f35009c69 --- /dev/null +++ b/docs/eventsource-names.md @@ -0,0 +1,41 @@ +# EventSource Names + +In a Sensor object, a `dependency` is defined as: + +```yaml +dependencies: + - name: test-dep + eventSourceName: webhook-example + eventName: example +``` + +The `eventSourceName` ad `eventName` might be confusing. Take the following +EventSource example, the `eventSourceName` and `eventName` are described as +below. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: + name: webhook-example # eventSourceName +spec: + webhook: + example: + port: "12000" + endpoint: /example # eventName + method: POST + example-foo: # eventName + port: "13000" + endpoint: /example2 + method: POST +``` + +## EventSourceName + +`eventSourceName` is the `name` of the dependent `EventSource` object, i.e. +`webhook-example` in the example above. + +## EventName + +`eventName` is the map key of a configured event. In the example above, +`eventName` could be `exmaple` or `example-foo`. diff --git a/docs/eventsource-with-multiple-events.md b/docs/eventsource-with-multiple-events.md new file mode 100644 index 0000000000..a386f6b320 --- /dev/null +++ b/docs/eventsource-with-multiple-events.md @@ -0,0 +1,78 @@ +# EventSource With Multiple Events + +![alpha](assets/alpha.svg) + +> v0.17.0 and after + +Multiple events can be configured in a single EventSource, they can be either +one event source type, or mixed event source types with some limitations. + +## Single EventSource Type + +A single type EventSource configuration: + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: + name: webhook +spec: + webhook: + example: + port: "12000" + endpoint: /example + method: POST + example-foo: + port: "13000" + endpoint: /example2 + method: POST +``` + +For the example above, there are 2 events configured in the EventSource named +`webhook`. Please use different `port` numbers for different events, this is the +limitation for multiple events configured in a `webhook` EventSource, this +limitation also applies to `webhook` extended event source types such as +`github`, `sns`. + +## Mixed EventSource Types + +EventSource is allowed to have mixed types of events configured. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: + name: mixed-sources +spec: + webhook: + webhook-example: # eventName + port: "12000" + endpoint: /example + method: POST + sns: + sns-example: # eventName + topicArn: arn:aws:sns:us-east-1:XXXXXXXX:test + webhook: + endpoint: "/" + port: "15000" + accessKey: + key: my-key + name: my-name + secretKey: + key: my-secret-key + name: my-secret-name + region: us-east-1 +``` + +However, there are some rules need to follow to do it: + +- `Rolling Update` types and `Recreate` types can not be configured together, + see [EventSource Deployment Strategies](eventsource-deployment-strategies.md). + +- Event Name (i.e. `webhook-example` and `sns-example` above, refer to + [EventSource Names](eventsource-names.md)) needs to be unique in the + EventSource, same `eventName` is not allowed even they are in different event + source types. + + The reason for that is, we use `eventSourceName` and `eventName` as the + dependency attributes in Sensor. diff --git a/pkg/apis/common/common.go b/pkg/apis/common/common.go index ec1bfa6a60..8e27625d74 100644 --- a/pkg/apis/common/common.go +++ b/pkg/apis/common/common.go @@ -51,14 +51,20 @@ var ( // RecreateStrategyEventSources refers to the list of event source types // that need to use Recreate strategy for its Deployment RecreateStrategyEventSources = []EventSourceType{ + CalendarEvent, KafkaEvent, PubSubEvent, AzureEventsHub, NATSEvent, MQTTEvent, + MinioEvent, EmitterEvent, NSQEvent, + PulsarEvent, ResourceEvent, + HDFSEvent, + FileEvent, + MinioEvent, } ) From be749747de55a5d8df360966a6a67857d8e47689 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Tue, 4 Aug 2020 18:26:54 -0700 Subject: [PATCH 2/8] log --- controllers/eventsource/validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/eventsource/validate.go b/controllers/eventsource/validate.go index 58062c2d9e..6f3b9168de 100644 --- a/controllers/eventsource/validate.go +++ b/controllers/eventsource/validate.go @@ -51,7 +51,7 @@ func ValidateEventSource(eventSource *v1alpha1.EventSource) error { if rollingUpdates > 0 && recreates > 0 { // We don't allow this as if we use recreate strategy for the deployment it will have downtime eventSource.Status.MarkSourcesNotProvided("InvalidEventSource", "Some types of event sources can not be put in one spec") - return errors.New("event sources with rolling update and recreate update strategy can not put together") + return errors.New("event sources with rolling update and recreate update strategy can not be put together") } eventSource.Status.MarkSourcesProvided() From 9ec42647fc10f5d5ca9c4ac8b069e9ba6ed25822 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Tue, 4 Aug 2020 19:30:39 -0700 Subject: [PATCH 3/8] fix test case --- controllers/eventsource/validate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/eventsource/validate_test.go b/controllers/eventsource/validate_test.go index c768f2a2d6..ceab5d5d3d 100644 --- a/controllers/eventsource/validate_test.go +++ b/controllers/eventsource/validate_test.go @@ -28,7 +28,7 @@ func TestValidate(t *testing.T) { testEventSource.Spec.Webhook = fakeWebhookEventSourceMap("test2") err := ValidateEventSource(testEventSource) assert.Error(t, err) - assert.Equal(t, "event sources with rolling update and recreate update strategy can not put together", err.Error()) + assert.Equal(t, "event sources with rolling update and recreate update strategy can not be put together", err.Error()) }) t.Run("validate bad mixed types eventsource - duplicated name", func(t *testing.T) { From e5ebc70e89ea4ac4fef336180353156f47bf0311 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Tue, 4 Aug 2020 19:32:35 -0700 Subject: [PATCH 4/8] comments --- docs/eventsource-names.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/eventsource-names.md b/docs/eventsource-names.md index 7f35009c69..da7f26560b 100644 --- a/docs/eventsource-names.md +++ b/docs/eventsource-names.md @@ -20,9 +20,9 @@ metadata: name: webhook-example # eventSourceName spec: webhook: - example: + example: # eventName port: "12000" - endpoint: /example # eventName + endpoint: /example method: POST example-foo: # eventName port: "13000" From 4c4edee82e5c1e64597dd8e3b0a9c7087e5d9674 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Tue, 4 Aug 2020 19:44:53 -0700 Subject: [PATCH 5/8] minor --- docs/eventsource-deployment-strategies.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/eventsource-deployment-strategies.md b/docs/eventsource-deployment-strategies.md index 6612cfb8d0..9c5ce6ebba 100644 --- a/docs/eventsource-deployment-strategies.md +++ b/docs/eventsource-deployment-strategies.md @@ -21,6 +21,11 @@ are different for different event sources. - Stripe - Webhook +### Replicas Of Rolling Update Types + +Deployment replica of these event source types respects the field of `replica` +in EventSource object, defaults to 1. + ## Recreate Strategy `Recreate` strategy is applied to the following EventSource types: @@ -40,7 +45,7 @@ are different for different event sources. - Resource - Calendar -### Replicas +### Replicas Of Recreate Types The field `replica` for EventSource of these `Recreate` types are ignored, the deployment is always created with `replica=1`. From 07f2656c369bbc5f5988dc19222d2fc524c54630 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Tue, 4 Aug 2020 19:45:58 -0700 Subject: [PATCH 6/8] minor again --- docs/eventsource-deployment-strategies.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/eventsource-deployment-strategies.md b/docs/eventsource-deployment-strategies.md index 9c5ce6ebba..0a9a35adcf 100644 --- a/docs/eventsource-deployment-strategies.md +++ b/docs/eventsource-deployment-strategies.md @@ -47,7 +47,7 @@ in EventSource object, defaults to 1. ### Replicas Of Recreate Types -The field `replica` for EventSource of these `Recreate` types are ignored, the +The field `replica` for EventSource of these `Recreate` types is ignored, the deployment is always created with `replica=1`. **Please DO NOT manually scale up the replicas, that will bring unexpected From d942a266c151a596e24f2a3f373e619dc282f851 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Tue, 4 Aug 2020 22:32:26 -0700 Subject: [PATCH 7/8] change --- docs/eventsource-deployment-strategies.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/eventsource-deployment-strategies.md b/docs/eventsource-deployment-strategies.md index 0a9a35adcf..9b886e5010 100644 --- a/docs/eventsource-deployment-strategies.md +++ b/docs/eventsource-deployment-strategies.md @@ -23,8 +23,8 @@ are different for different event sources. ### Replicas Of Rolling Update Types -Deployment replica of these event source types respects the field of `replica` -in EventSource object, defaults to 1. +Deployment replica of these event source types respects `spec.replica` in the +EventSource object, defaults to 1. ## Recreate Strategy @@ -47,8 +47,8 @@ in EventSource object, defaults to 1. ### Replicas Of Recreate Types -The field `replica` for EventSource of these `Recreate` types is ignored, the -deployment is always created with `replica=1`. +`spec.replica` in the `Recreate` types EventSources is ignored, the deployment +is always created with `replica=1`. -**Please DO NOT manually scale up the replicas, that will bring unexpected +**Please DO NOT manually scale up the replicas, that will lead to unexpected behaviors!** From d63efb6351c33e6cadd0b3472d6d7825bda86dce Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Wed, 5 Aug 2020 08:17:30 -0700 Subject: [PATCH 8/8] update --- docs/eventsource-deployment-strategies.md | 2 +- pkg/apis/common/common.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/eventsource-deployment-strategies.md b/docs/eventsource-deployment-strategies.md index 9b886e5010..e40dbd6b9f 100644 --- a/docs/eventsource-deployment-strategies.md +++ b/docs/eventsource-deployment-strategies.md @@ -11,7 +11,6 @@ are different for different event sources. `Rolling Update` strategy is applied to the following EventSource types: -- AMQP - AWS SNS - AWS SQS - Github @@ -30,6 +29,7 @@ EventSource object, defaults to 1. `Recreate` strategy is applied to the following EventSource types: +- AMQP - Azure Events Hub - Kafka - GCP PubSub diff --git a/pkg/apis/common/common.go b/pkg/apis/common/common.go index 8e27625d74..852182e3b2 100644 --- a/pkg/apis/common/common.go +++ b/pkg/apis/common/common.go @@ -51,6 +51,7 @@ var ( // RecreateStrategyEventSources refers to the list of event source types // that need to use Recreate strategy for its Deployment RecreateStrategyEventSources = []EventSourceType{ + AMQPEvent, CalendarEvent, KafkaEvent, PubSubEvent, @@ -61,10 +62,10 @@ var ( EmitterEvent, NSQEvent, PulsarEvent, + RedisEvent, ResourceEvent, HDFSEvent, FileEvent, - MinioEvent, } )