From 4803e76abc8c26c32ed585a3a7a8684ff01a1495 Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Thu, 25 Jul 2024 13:43:21 -0400 Subject: [PATCH] feat: Add support for sending/receiving gzip event payloads (#364) --- config/config.go | 11 +- config/config_validation.go | 12 +++ config/test_data_configs_valid_test.go | 33 +++++- docs/configuration.md | 16 +-- go.mod | 17 ++-- go.sum | 37 ++++--- internal/events/event-relay.go | 9 +- internal/events/event-relay_test.go | 43 ++++++-- internal/events/event_publisher.go | 11 +- internal/events/event_publisher_test.go | 47 +++++++-- internal/events/summarizing-relay.go | 9 +- internal/events/summarizing-relay_test.go | 32 ++++-- internal/middleware/middleware.go | 19 ++++ internal/relayenv/env_context_impl.go | 2 +- internal/relayenv/env_context_impl_test.go | 11 +- internal/util/reader.go | 111 +++++++++++++++++++++ internal/util/reader_test.go | 72 +++++++++++++ internal/util/util.go | 12 +++ relay/autoconfig_key_change_test.go | 7 +- relay/endpoints_events_test.go | 5 +- relay/relay_routes.go | 14 ++- 21 files changed, 447 insertions(+), 83 deletions(-) create mode 100644 internal/util/reader.go create mode 100644 internal/util/reader_test.go diff --git a/config/config.go b/config/config.go index 08af54f0..0ff81832 100644 --- a/config/config.go +++ b/config/config.go @@ -184,11 +184,12 @@ type OfflineModeConfig struct { // variables, individual fields are not documented here; instead, see the `README.md` section on // configuration. type EventsConfig struct { - EventsURI ct.OptURLAbsolute `conf:"EVENTS_HOST"` - SendEvents bool `conf:"USE_EVENTS"` - FlushInterval ct.OptDuration `conf:"EVENTS_FLUSH_INTERVAL"` - Capacity ct.OptIntGreaterThanZero `conf:"EVENTS_CAPACITY"` - InlineUsers bool `conf:"EVENTS_INLINE_USERS"` + EventsURI ct.OptURLAbsolute `conf:"EVENTS_HOST"` + SendEvents bool `conf:"USE_EVENTS"` + FlushInterval ct.OptDuration `conf:"EVENTS_FLUSH_INTERVAL"` + Capacity ct.OptIntGreaterThanZero `conf:"EVENTS_CAPACITY"` + InlineUsers bool `conf:"EVENTS_INLINE_USERS"` + MaxInboundPayloadSize ct.OptBase2Bytes `conf:"EVENTS_MAX_INBOUND_PAYLOAD_SIZE"` } // RedisConfig configures the optional Redis integration. diff --git a/config/config_validation.go b/config/config_validation.go index 9708c076..84996dda 100644 --- a/config/config_validation.go +++ b/config/config_validation.go @@ -16,6 +16,7 @@ var ( errFileDataWithAutoConf = errors.New("cannot specify both auto-configuration key and file data source") errOfflineModePropertiesWithNoFile = errors.New("must specify offline mode filename if other offline mode properties are set") errOfflineModeWithEnvironments = errors.New("cannot configure specific environments if offline mode is enabled") + errMaxInboundPayloadSize = errors.New("max inbound payload size must be greater than zero") errAutoConfWithoutDBDisambig = errors.New(`when using auto-configuration with database storage, database prefix (or,` + ` if using DynamoDB, table name) must be specified and must contain "` + AutoConfigEnvironmentIDPlaceholder + `"`) errRedisURLWithHostAndPort = errors.New("please specify Redis URL or host/port, but not both") @@ -80,6 +81,7 @@ func ValidateConfig(c *Config, loggers ldlog.Loggers) error { validateConfigFilters(&result, c) validateOfflineMode(&result, c) validateCredentialCleanupInterval(&result, c) + validateMaxInboundPayloadSize(&result, c) return result.GetError() } @@ -206,6 +208,16 @@ func validateCredentialCleanupInterval(result *ct.ValidationResult, c *Config) { } } } + +func validateMaxInboundPayloadSize(result *ct.ValidationResult, c *Config) { + if c.Events.MaxInboundPayloadSize.IsDefined() { + size := c.Events.MaxInboundPayloadSize.GetOrElse(0) + if size <= 0 { + result.AddError(nil, errMaxInboundPayloadSize) + } + } +} + func validateConfigDatabases(result *ct.ValidationResult, c *Config, loggers ldlog.Loggers) { normalizeRedisConfig(result, c) diff --git a/config/test_data_configs_valid_test.go b/config/test_data_configs_valid_test.go index b5d875a0..5e085823 100644 --- a/config/test_data_configs_valid_test.go +++ b/config/test_data_configs_valid_test.go @@ -65,6 +65,9 @@ func makeValidConfigs() []testDataValidConfig { makeValidConfigExplicitOldDefaultBaseURI(), makeValidConfigAutoConfig(), makeValidConfigAutoConfigWithDatabase(), + makeValidConfigMaxInboundPayloadSize("50KiB"), + makeValidConfigMaxInboundPayloadSize("7MiB"), + makeValidConfigMaxInboundPayloadSize("10GiB"), makeValidConfigOfflineModeMinimal(), makeValidConfigOfflineModeWithMonitoringInterval("100ms"), makeValidConfigOfflineModeWithMonitoringInterval("1s"), @@ -118,11 +121,12 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { ExpiredCredentialCleanupInterval: ct.NewOptDuration(1 * time.Minute), } c.Events = EventsConfig{ - SendEvents: true, - EventsURI: newOptURLAbsoluteMustBeValid("http://events"), - FlushInterval: ct.NewOptDuration(120 * time.Second), - Capacity: mustOptIntGreaterThanZero(500), - InlineUsers: true, + SendEvents: true, + EventsURI: newOptURLAbsoluteMustBeValid("http://events"), + FlushInterval: ct.NewOptDuration(120 * time.Second), + Capacity: mustOptIntGreaterThanZero(500), + InlineUsers: true, + MaxInboundPayloadSize: ct.OptBase2Bytes{}, } c.Environment = map[string]*EnvConfig{ "earth": { @@ -356,6 +360,25 @@ FileDataSource = my-file-path return c } +func makeValidConfigMaxInboundPayloadSize(size string) testDataValidConfig { + bytes, err := ct.NewOptBase2BytesFromString(size) + if err != nil { + panic(err) + } + c := testDataValidConfig{name: "file data properties"} + c.makeConfig = func(c *Config) { + c.Events.MaxInboundPayloadSize = bytes + } + c.envVars = map[string]string{ + "EVENTS_MAX_INBOUND_PAYLOAD_SIZE": size, + } + c.fileContent = ` +[Events] +MaxInboundPayloadSize = ` + size + + return c +} + func MustOptDurationFromString(duration string) ct.OptDuration { opt, err := ct.NewOptDurationFromString(duration) if err != nil { diff --git a/docs/configuration.md b/docs/configuration.md index f7e4a436..becf5250 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -114,15 +114,17 @@ Note that the last three properties have the same meanings and the same environm To learn more, read [Forwarding events](./events.md). -| Property in file | Environment var | Type | Default | Description | -|------------------|-------------------------|:--------:|:--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `sendEvents` | `USE_EVENTS` | Boolean | `false` | When enabled, the Relay Proxy will send analytic events it receives to LaunchDarkly, unless offline mode is enabled. | -| `eventsUri` | `EVENTS_HOST` | URI | _(7)_ | URI for the LaunchDarkly events service | -| `flushInterval` | `EVENTS_FLUSH_INTERVAL` | Duration | `5s` | Controls how long the SDK buffers events before sending them back to our server. If your server generates many events per second, we suggest decreasing the flush interval and/or increasing capacity to meet your needs. | -| `capacity` | `EVENTS_CAPACITY` | Number | `1000` | Maximum number of events to accumulate for each flush interval. | -| `inlineUsers` | `EVENTS_INLINE_USERS` | Boolean | `false` | When enabled, individual events (if full event tracking is enabled for the feature flag) will contain all non-private user attributes. | +| Property in file | Environment var | Type | Default | Description | +|-------------------------------|--------------------------------------|:--------:|:--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `sendEvents` | `USE_EVENTS` | Boolean | `false` | When enabled, the Relay Proxy will send analytic events it receives to LaunchDarkly, unless offline mode is enabled. | +| `eventsUri` | `EVENTS_HOST` | URI | _(7)_ | URI for the LaunchDarkly events service | +| `flushInterval` | `EVENTS_FLUSH_INTERVAL` | Duration | `5s` | Controls how long the SDK buffers events before sending them back to our server. If your server generates many events per second, we suggest decreasing the flush interval and/or increasing capacity to meet your needs. | +| `capacity` | `EVENTS_CAPACITY` | Number | `1000` | Maximum number of events to accumulate for each flush interval. | +| `inlineUsers` | `EVENTS_INLINE_USERS` | Boolean | `false` | When enabled, individual events (if full event tracking is enabled for the feature flag) will contain all non-private user attributes. | +| `maxInboundPayloadSize` | `EVENTS_MAX_INBOUND_PAYLOAD_SIZE` | Unit | _(8)_ | Maximum size of an event payload the Relay Proxy will accept from an SDK. | _(7)_ See note _(1)_ above. The default value for `eventsUri` is `https://events.launchdarkly.com`. +_(8)_ The `maxInboundPayloadSize` setting is used to limit the size of the payload that the Relay Proxy will accept from an SDK. This is an optional safety feature to prevent the Relay Proxy from being overwhelmed by a very large payload. The default value is `0B` which provides no restriction on the payload size. The value should be a number followed by a unit: `B` for bytes, `KiB` for kibibytes, `MiB` for mebibytes, `GiB` for gibibytes, `TiB` for tebibytes, `PiB` for pebibytes, or `EiB` for exbibytes. For example, `100MiB` is 100 mebibytes. ### File section: `[Environment "NAME"]` diff --git a/go.mod b/go.mod index b2e6fe93..8f158c05 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/launchdarkly/ld-relay/v8 -go 1.20 +go 1.21 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 @@ -12,7 +12,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.0 github.com/cyphar/filepath-securejoin v0.2.4 github.com/fatih/color v1.15.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-redis/redis/v8 v8.11.5 github.com/gomodule/redigo v1.8.9 github.com/google/uuid v1.5.0 // indirect @@ -25,15 +25,15 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/kardianos/minwinsvc v1.0.2 github.com/launchdarkly/eventsource v1.7.1 - github.com/launchdarkly/go-configtypes v1.1.0 + github.com/launchdarkly/go-configtypes v1.2.0 github.com/launchdarkly/go-jsonstream/v3 v3.0.0 github.com/launchdarkly/go-sdk-common/v3 v3.1.0 - github.com/launchdarkly/go-sdk-events/v3 v3.2.0 + github.com/launchdarkly/go-sdk-events/v3 v3.4.0 github.com/launchdarkly/go-server-sdk-consul/v3 v3.0.0 github.com/launchdarkly/go-server-sdk-dynamodb/v4 v4.0.0 github.com/launchdarkly/go-server-sdk-evaluation/v3 v3.0.0 github.com/launchdarkly/go-server-sdk-redis-redigo/v3 v3.0.0 - github.com/launchdarkly/go-server-sdk/v7 v7.1.0 + github.com/launchdarkly/go-server-sdk/v7 v7.6.0 github.com/launchdarkly/go-test-helpers/v3 v3.0.2 github.com/launchdarkly/opencensus-go-exporter-stackdriver v0.14.2 github.com/pborman/uuid v1.2.1 @@ -59,7 +59,10 @@ require ( golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb ) -require github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e +require ( + github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 + github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e +) require ( cloud.google.com/go/compute v1.23.3 // indirect @@ -123,8 +126,6 @@ require ( google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/DataDog/dd-trace-go.v1 v1.56.1 // indirect - gopkg.in/launchdarkly/go-jsonstream.v1 v1.0.1 // indirect - gopkg.in/launchdarkly/go-sdk-common.v2 v2.5.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 16e0c12c..aebbaa57 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,7 @@ cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKP cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -63,6 +64,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= @@ -197,6 +200,7 @@ github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs0 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -210,6 +214,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -244,8 +249,10 @@ github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:Fecb github.com/hashicorp/consul/api v1.25.1 h1:CqrdhYzc8XZuPnhIYZWH45toM0LB9ZeYr/gvpLVI3PE= github.com/hashicorp/consul/api v1.25.1/go.mod h1:iiLVwR/htV7mas/sy0O+XSuEnrdBUUydemjxcUrAt4g= github.com/hashicorp/consul/sdk v0.14.1 h1:ZiwE2bKb+zro68sWzZ1SgHF3kRMBZ94TwOCFRF4ylPs= +github.com/hashicorp/consul/sdk v0.14.1/go.mod h1:vFt03juSzocLRFo59NkeQHHmQa6+g7oU0pfzdI1mUhg= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -256,9 +263,11 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= @@ -269,6 +278,7 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -309,17 +319,19 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e h1:PZ8SXmC5B/jTc8FfrWfSGNjy0ieGwqcKPjPV4vMtUqM= github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e/go.mod h1:cQRkOAs0LGcfIs6RSsHNqwhzItUZooyhpqPv0hgiQZM= github.com/launchdarkly/ccache v1.1.0 h1:voD1M+ZJXR3MREOKtBwgTF9hYHl1jg+vFKS/+VAkR2k= github.com/launchdarkly/ccache v1.1.0/go.mod h1:TlxzrlnzvYeXiLHmesMuvoZetu4Z97cV1SsdqqBJi1Q= github.com/launchdarkly/eventsource v1.7.1 h1:StoRQeiPyrcQIXjlQ7b5jWMzHW4p+GGczN2r2oBhujg= github.com/launchdarkly/eventsource v1.7.1/go.mod h1:LHxSeb4OnqznNZxCSXbFghxS/CjIQfzHovNoAqbO/Wk= -github.com/launchdarkly/go-configtypes v1.1.0 h1:Qsp/q607eXPJqUcQdSdAs0+vDG+luqVCmenS9a302EI= -github.com/launchdarkly/go-configtypes v1.1.0/go.mod h1:AL0LzOSTYBAEeZJ8JpBWlz7BbHhxTQvOTcn7ZBgx7IA= +github.com/launchdarkly/go-configtypes v1.2.0 h1:Jffn87daoeJZBGwKNgaf7UcIZIk1DUVIo/HPauXHwlM= +github.com/launchdarkly/go-configtypes v1.2.0/go.mod h1:HtmuhT0J4NUTtvlVIlvc84NpmsJ1mWlcrajbM7R8K3Q= github.com/launchdarkly/go-jsonstream/v3 v3.0.0 h1:qJF/WI09EUJ7kSpmP5d1Rhc81NQdYUhP17McKfUq17E= github.com/launchdarkly/go-jsonstream/v3 v3.0.0/go.mod h1:/1Gyml6fnD309JOvunOSfyysWbZ/ZzcA120gF/cQtC4= github.com/launchdarkly/go-ntlm-proxy-auth v1.0.1 h1:Iz5cg9mB/0vt5llZE+J0iGQ5+O/U8CWuRUUR7Ou8eNM= @@ -328,8 +340,8 @@ github.com/launchdarkly/go-ntlmssp v1.0.1 h1:snB77118TQvf9tfHrkSyrIop/UX5e5VD2D2 github.com/launchdarkly/go-ntlmssp v1.0.1/go.mod h1:/cq3t2JyALD7GdVF5BEWcEuGlIGa44FZ4v4CVk7vuCY= github.com/launchdarkly/go-sdk-common/v3 v3.1.0 h1:KNCP5rfkOt/25oxGLAVgaU1BgrZnzH9Y/3Z6I8bMwDg= github.com/launchdarkly/go-sdk-common/v3 v3.1.0/go.mod h1:mXFmDGEh4ydK3QilRhrAyKuf9v44VZQWnINyhqbbOd0= -github.com/launchdarkly/go-sdk-events/v3 v3.2.0 h1:FUby/4cUSVDghCkFDpvy+7vZlIW4+CK95HjQnuqGXVs= -github.com/launchdarkly/go-sdk-events/v3 v3.2.0/go.mod h1:oepYWQ2RvvjfL2WxkE1uJJIuRsIMOP4WIVgUpXRPcNI= +github.com/launchdarkly/go-sdk-events/v3 v3.4.0 h1:22sVSEDEXpdOEK3UBtmThwsUHqc+cbbe/pJfsliBAA4= +github.com/launchdarkly/go-sdk-events/v3 v3.4.0/go.mod h1:oepYWQ2RvvjfL2WxkE1uJJIuRsIMOP4WIVgUpXRPcNI= github.com/launchdarkly/go-semver v1.0.2 h1:sYVRnuKyvxlmQCnCUyDkAhtmzSFRoX6rG2Xa21Mhg+w= github.com/launchdarkly/go-semver v1.0.2/go.mod h1:xFmMwXba5Mb+3h72Z+VeSs9ahCvKo2QFUTHRNHVqR28= github.com/launchdarkly/go-server-sdk-consul/v3 v3.0.0 h1:AXmmU4rsMxdA75o4a9p+7Pl3SzdfUCLIw7CM7pBRifE= @@ -340,15 +352,15 @@ github.com/launchdarkly/go-server-sdk-evaluation/v3 v3.0.0 h1:nQbR1xCpkdU9Z71FI2 github.com/launchdarkly/go-server-sdk-evaluation/v3 v3.0.0/go.mod h1:cwk7/7SzNB2wZbCZS7w2K66klMLBe3NFM3/qd3xnsRc= github.com/launchdarkly/go-server-sdk-redis-redigo/v3 v3.0.0 h1:ItkPbTEzz0ObNzIpA3DfPqgEgeNyqno/Lnfd7BjC0Ns= github.com/launchdarkly/go-server-sdk-redis-redigo/v3 v3.0.0/go.mod h1:ho3n0ML1YbV0QRnidDNF9ooFIC66FiVzZGW0u4behG0= -github.com/launchdarkly/go-server-sdk/v7 v7.1.0 h1:gv7LXMuP0v4/iCcYE4F/4FFHHHo+K1OftJTxgm9WXQE= -github.com/launchdarkly/go-server-sdk/v7 v7.1.0/go.mod h1:+VdDZzCw8eyhuakZAJciiB6NjfltApenvIRTuTiqtFE= +github.com/launchdarkly/go-server-sdk/v7 v7.6.0 h1:151suUEHdGME1LkgOt+rqQfxfo1nDEF2EEA90adEarY= +github.com/launchdarkly/go-server-sdk/v7 v7.6.0/go.mod h1:2xLqMTxh5cVEliZUKt66uOQR/8vcc9ObOVmjlhSVY/0= github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw= github.com/launchdarkly/go-test-helpers/v2 v2.3.2 h1:WX6qSzt7v8xz6d94nVcoil9ljuLTC/6OzQt0MhWYxsQ= +github.com/launchdarkly/go-test-helpers/v2 v2.3.2/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw= github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU= github.com/launchdarkly/go-test-helpers/v3 v3.0.2/go.mod h1:u2ZvJlc/DDJTFrshWW50tWMZHLVYXofuSHUfTU/eIwM= github.com/launchdarkly/opencensus-go-exporter-stackdriver v0.14.2 h1:ojocsFdk2ySSFqIAmtJxOQqF2aQo3q1etd+VTb5V9pA= github.com/launchdarkly/opencensus-go-exporter-stackdriver v0.14.2/go.mod h1:E0xXDxC1FNMuovDLNlFQCY8dzQs3WoKyML8R7bugsOI= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -372,6 +384,7 @@ github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQth github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= +github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -386,7 +399,9 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -459,6 +474,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= +github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -869,15 +885,12 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/launchdarkly/go-jsonstream.v1 v1.0.1 h1:aZHvMDAS+M6/0sRMkDBQ8MyLGsTQrNgN5evu5e8UYpQ= -gopkg.in/launchdarkly/go-jsonstream.v1 v1.0.1/go.mod h1:YefdBjfITIP8D9BJLVbssFctHkJnQXhv+TiRdTV0Jr4= -gopkg.in/launchdarkly/go-sdk-common.v2 v2.0.0/go.mod h1:4l1+/AtknK5Sx6YTO9XDqrCbAXj8FgwpI2U/x6ZBIM4= -gopkg.in/launchdarkly/go-sdk-common.v2 v2.5.1 h1:RqucG3hCU/GAupuEyVXXPf0Hz3F4InyhiFR2sfUbgBs= -gopkg.in/launchdarkly/go-sdk-common.v2 v2.5.1/go.mod h1:P2+C6CHteys+lEDd6298QszCsMhjdYrfzBd6dg//CHA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/events/event-relay.go b/internal/events/event-relay.go index 3ecdb201..fc9ede18 100644 --- a/internal/events/event-relay.go +++ b/internal/events/event-relay.go @@ -129,10 +129,11 @@ func (d *diagnosticEventEndpointDispatcher) dispatch(w http.ResponseWriter, req d.loggers.Debugf("Received diagnostic event to be proxied to %s/%s", d.baseURI, d.uriPath) sendConfig := ldevents.EventSenderConfiguration{ - Client: d.httpClient, - BaseURI: d.baseURI, - BaseHeaders: func() http.Header { return req.Header }, - Loggers: d.loggers, + Client: d.httpClient, + BaseURI: d.baseURI, + BaseHeaders: func() http.Header { return req.Header }, + Loggers: d.loggers, + EnableCompression: true, } _ = ldevents.SendEventDataWithRetry(sendConfig, ldevents.DiagnosticEventDataKind, d.uriPath, body, 1) }) diff --git a/internal/events/event-relay_test.go b/internal/events/event-relay_test.go index dd8810b4..619bb12a 100644 --- a/internal/events/event-relay_test.go +++ b/internal/events/event-relay_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/util" "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/basictypes" @@ -178,7 +179,10 @@ func TestVerbatimEventHandlers(t *testing.T) { assert.Equal(t, e.analyticsPath, r.Request.URL.Path) assert.Equal(t, e.authKey, r.Request.Header.Get("Authorization")) assert.Equal(t, strconv.Itoa(CurrentEventsSchemaVersion), r.Request.Header.Get(EventSchemaHeader)) - assert.Equal(t, eventPayloadForVerbatimOnly, string(r.Body)) + + uncompressed, err := util.DecompressGzipData(r.Body) + require.NoError(t, err) + assert.Equal(t, eventPayloadForVerbatimOnly, string(uncompressed)) }) }) } @@ -208,16 +212,33 @@ func TestVerbatimEventHandlers(t *testing.T) { helpers.RequireValue(t, p.requestsCh, time.Second), helpers.RequireValue(t, p.requestsCh, time.Second), } - sort.Slice(received, func(i, j int) bool { return string(received[i].Body) < string(received[j].Body) }) + + type receivedPayload struct { + Request *http.Request + UncompressedBody string + } + + receivedPayloads := make([]receivedPayload, 0, len(received)) + for _, r := range received { + uncompressed, err := util.DecompressGzipData([]byte(r.Body)) + require.NoError(t, err) + receivedPayloads = append(receivedPayloads, receivedPayload{Request: r.Request, UncompressedBody: string(uncompressed)}) + } + + sort.Slice(receivedPayloads, func(i, j int) bool { + return receivedPayloads[i].UncompressedBody < receivedPayloads[j].UncompressedBody + }) + + for _, r := range receivedPayloads { assert.Equal(t, "POST", r.Request.Method) assert.Equal(t, e.analyticsPath, r.Request.URL.Path) assert.Equal(t, e.authKey, r.Request.Header.Get("Authorization")) } - assert.Equal(t, "3", received[0].Request.Header.Get(EventSchemaHeader)) - assert.Equal(t, `["fake-event-v3-1","fake-event-v3-2","fake-event-v3-3"]`, string(received[0].Body)) - assert.Equal(t, "4", received[1].Request.Header.Get(EventSchemaHeader)) - assert.Equal(t, `["fake-event-v4-1","fake-event-v4-2"]`, string(received[1].Body)) + assert.Equal(t, "3", receivedPayloads[0].Request.Header.Get(EventSchemaHeader)) + assert.Equal(t, `["fake-event-v3-1","fake-event-v3-2","fake-event-v3-3"]`, receivedPayloads[0].UncompressedBody) + assert.Equal(t, "4", receivedPayloads[1].Request.Header.Get(EventSchemaHeader)) + assert.Equal(t, `["fake-event-v4-1","fake-event-v4-2"]`, receivedPayloads[1].UncompressedBody) }) }) } @@ -246,7 +267,10 @@ func TestSummarizingEventHandlers(t *testing.T) { assert.Equal(t, e.analyticsPath, r.Request.URL.Path) assert.Equal(t, e.authKey, r.Request.Header.Get("Authorization")) assert.Equal(t, strconv.Itoa(CurrentEventsSchemaVersion), r.Request.Header.Get(EventSchemaHeader)) - m.In(t).Assert(r.Body, m.JSONStrEqual(summarizeEventsParams.expectedEventsJSON)) + + uncompressed, err := util.DecompressGzipData(r.Body) + require.NoError(t, err) + m.In(t).Assert(uncompressed, m.JSONStrEqual(summarizeEventsParams.expectedEventsJSON)) }) }) } @@ -270,7 +294,10 @@ func TestDiagnosticEventForwarding(t *testing.T) { assert.Equal(t, e.diagnosticPath, r.Request.URL.Path) assert.Equal(t, "fake-auth", r.Request.Header.Get("Authorization")) assert.Equal(t, "fake-user-agent", r.Request.Header.Get("User-Agent")) - assert.Equal(t, eventPayloadForVerbatimOnly, string(r.Body)) + + uncompressed, err := util.DecompressGzipData([]byte(r.Body)) + require.NoError(t, err) + assert.Equal(t, eventPayloadForVerbatimOnly, string(uncompressed)) }) }) } diff --git a/internal/events/event_publisher.go b/internal/events/event_publisher.go index ae7e0258..d286cecb 100644 --- a/internal/events/event_publisher.go +++ b/internal/events/event_publisher.go @@ -343,11 +343,12 @@ func (p *HTTPEventPublisher) flush() { // and error logging, in its SendEventData method. Retries could cause this call to block for a while, // so it's run on a separate goroutine. sendConfig := ldevents.EventSenderConfiguration{ - Client: p.client, - BaseURI: p.baseURI, - BaseHeaders: getBaseHeaders, - SchemaVersion: schemaVersion, - Loggers: p.loggers, + Client: p.client, + BaseURI: p.baseURI, + BaseHeaders: getBaseHeaders, + SchemaVersion: schemaVersion, + Loggers: p.loggers, + EnableCompression: true, } result := ldevents.SendEventDataWithRetry(sendConfig, ldevents.AnalyticsEventDataKind, p.uriPath, payload, count) p.wg.Done() diff --git a/internal/events/event_publisher_test.go b/internal/events/event_publisher_test.go index 5dff7245..2155b434 100644 --- a/internal/events/event_publisher_test.go +++ b/internal/events/event_publisher_test.go @@ -11,6 +11,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" + "github.com/launchdarkly/ld-relay/v8/internal/util" "github.com/launchdarkly/go-sdk-common/v3/ldlog" "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" @@ -45,7 +46,10 @@ func TestHTTPEventPublisherSimple(t *testing.T) { assert.Equal(t, "/bulk", r.Request.URL.Path) assert.Equal(t, string(testSDKKey), r.Request.Header.Get("Authorization")) assert.Equal(t, strconv.Itoa(CurrentEventsSchemaVersion), r.Request.Header.Get(EventSchemaHeader)) - m.In(t).Assert(r.Body, m.JSONStrEqual(`["hello", "hello again"]`)) + + uncompressed, err := util.DecompressGzipData([]byte(r.Body)) + assert.NoError(t, err) + m.In(t).Assert(uncompressed, m.JSONStrEqual(`["hello", "hello again"]`)) }) } @@ -77,19 +81,28 @@ func TestHTTPEventPublisherMultiQueuesWithMetadata(t *testing.T) { assert.Equal(t, string(testSDKKey), r0.Request.Header.Get("Authorization")) assert.Equal(t, "3", r0.Request.Header.Get(EventSchemaHeader)) assert.Equal(t, "a", r0.Request.Header.Get(TagsHeader)) - m.In(t).Assert(r0.Body, m.JSONStrEqual(`["also this"]`)) + + uncompressed0, err := util.DecompressGzipData(r0.Body) + assert.NoError(t, err) + m.In(t).Assert(uncompressed0, m.JSONStrEqual(`["also this"]`)) assert.Equal(t, "/bulk", received[0].Request.URL.Path) assert.Equal(t, string(testSDKKey), r1.Request.Header.Get("Authorization")) assert.Equal(t, strconv.Itoa(CurrentEventsSchemaVersion), r1.Request.Header.Get(EventSchemaHeader)) assert.Equal(t, "a", r1.Request.Header.Get(TagsHeader)) - m.In(t).Assert(r1.Body, m.JSONStrEqual(`["hello", "hello again"]`)) + + uncompressed1, err := util.DecompressGzipData(r1.Body) + assert.NoError(t, err) + m.In(t).Assert(uncompressed1, m.JSONStrEqual(`["hello", "hello again"]`)) assert.Equal(t, "/bulk", r2.Request.URL.Path) assert.Equal(t, string(testSDKKey), r2.Request.Header.Get("Authorization")) assert.Equal(t, strconv.Itoa(CurrentEventsSchemaVersion), r2.Request.Header.Get(EventSchemaHeader)) assert.Equal(t, "b", r2.Request.Header.Get(TagsHeader)) - m.In(t).Assert(r2.Body, m.JSONStrEqual(`["ok", "thanks"]`)) + + uncompressed2, err := util.DecompressGzipData(r2.Body) + assert.NoError(t, err) + m.In(t).Assert(uncompressed2, m.JSONStrEqual(`["ok", "thanks"]`)) }) } @@ -107,7 +120,10 @@ func TestHTTPEventPublisherOptionURIPath(t *testing.T) { assert.Equal(t, "/special-path", r.Request.URL.Path) assert.Equal(t, string(testSDKKey), r.Request.Header.Get("Authorization")) assert.Equal(t, strconv.Itoa(CurrentEventsSchemaVersion), r.Request.Header.Get(EventSchemaHeader)) - m.In(t).Assert(r.Body, m.JSONStrEqual(`["hello"]`)) + + uncompressed, err := util.DecompressGzipData(r.Body) + assert.NoError(t, err) + m.In(t).Assert(uncompressed, m.JSONStrEqual(`["hello"]`)) }) } @@ -130,7 +146,10 @@ func TestHTTPPublisherAutomaticFlush(t *testing.T) { publisher.Publish(EventPayloadMetadata{}, json.RawMessage(`"hello"`)) r := helpers.RequireValue(t, requestsCh, time.Second) assert.Equal(t, "/bulk", r.Request.URL.Path) - m.In(t).Assert(r.Body, m.JSONStrEqual(`["hello"]`)) + + uncompressed, err := util.DecompressGzipData(r.Body) + assert.NoError(t, err) + m.In(t).Assert(uncompressed, m.JSONStrEqual(`["hello"]`)) assert.Equal(t, strconv.Itoa(CurrentEventsSchemaVersion), r.Request.Header.Get(EventSchemaHeader)) }) } @@ -162,7 +181,11 @@ func TestHTTPEventPublisherCapacity(t *testing.T) { r := helpers.RequireValue(t, requestsCh, time.Second) assert.Equal(t, "/bulk", r.Request.URL.Path) assert.Equal(t, strconv.Itoa(CurrentEventsSchemaVersion), r.Request.Header.Get(EventSchemaHeader)) - m.In(t).Assert(r.Body, m.JSONStrEqual(`["hello"]`)) + + uncompressed, err := util.DecompressGzipData(r.Body) + assert.NoError(t, err) + + m.In(t).Assert(uncompressed, m.JSONStrEqual(`["hello"]`)) }) } @@ -182,10 +205,16 @@ func TestHTTPEventPublisherErrorRetry(t *testing.T) { timeStart := time.Now() publisher.Flush() req1 := helpers.RequireValue(t, requestsCh, time.Second*5) + uncompressed1, err := util.DecompressGzipData(req1.Body) + assert.NoError(t, err) + req2 := helpers.RequireValue(t, requestsCh, time.Second*5) + uncompressed2, err := util.DecompressGzipData(req2.Body) + assert.NoError(t, err) + elapsed := time.Since(timeStart) - assert.Equal(t, []byte(`["hello"]`), req1.Body) - assert.Equal(t, req1.Body, req2.Body) + assert.Equal(t, []byte(`["hello"]`), uncompressed1) + assert.Equal(t, uncompressed1, uncompressed2) assert.GreaterOrEqual(t, int64(elapsed), int64(time.Second)) // There were two failures, so it should not have retried again after that (should not reach successHandler) diff --git a/internal/events/summarizing-relay.go b/internal/events/summarizing-relay.go index d0e9b02b..d7dbcdaf 100644 --- a/internal/events/summarizing-relay.go +++ b/internal/events/summarizing-relay.go @@ -295,10 +295,11 @@ func makeEventSender( } return &eventSenderWithOverridePath{ config: ldevents.EventSenderConfiguration{ - Client: httpClient, - BaseURI: eventsURI, - BaseHeaders: func() http.Header { return headers }, - Loggers: loggers, + Client: httpClient, + BaseURI: eventsURI, + BaseHeaders: func() http.Header { return headers }, + Loggers: loggers, + EnableCompression: true, }, remotePath: remotePath, } diff --git a/internal/events/summarizing-relay_test.go b/internal/events/summarizing-relay_test.go index 3e2357dd..6ca5fa88 100644 --- a/internal/events/summarizing-relay_test.go +++ b/internal/events/summarizing-relay_test.go @@ -11,6 +11,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/basictypes" st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/util" ldevents "github.com/launchdarkly/go-sdk-events/v3" helpers "github.com/launchdarkly/go-test-helpers/v3" @@ -56,7 +57,10 @@ func TestSummarizeEvents(t *testing.T) { p.dispatcher.flush() payload := expectSummarizedPayload(t, p.requestsCh) - m.In(t).Assert(payload, m.JSONStrEqual(ep.expectedEventsJSON)) + + uncompressed, err := util.DecompressGzipData([]byte(payload)) + require.NoError(t, err) + m.In(t).Assert(uncompressed, m.JSONStrEqual(ep.expectedEventsJSON)) }) }) } @@ -97,12 +101,18 @@ func TestSummarizingRelayProcessesEventsSeparatelyForDifferentTags(t *testing.T) assert.Equal(t, "tags1", request1.Request.Header.Get(TagsHeader)) assert.Equal(t, "tags2", request2.Request.Header.Get(TagsHeader)) - m.In(t).Assert(json.RawMessage(request1.Body), m.JSONArray().Should(m.ItemsInAnyOrder( + decompressedBody1, err := util.DecompressGzipData(request1.Body) + assert.NoError(t, err) + + decompressedBody2, err := util.DecompressGzipData(request2.Body) + assert.NoError(t, err) + + m.In(t).Assert(json.RawMessage(decompressedBody1), m.JSONArray().Should(m.ItemsInAnyOrder( m.MapIncluding(m.KV("kind", m.Equal("index"))), m.MapIncluding(m.KV("kind", m.Equal("custom")), m.KV("key", m.Equal("eventkey1a"))), m.MapIncluding(m.KV("kind", m.Equal("custom")), m.KV("key", m.Equal("eventkey1b"))), ))) - m.In(t).Assert(json.RawMessage(request2.Body), m.JSONArray().Should(m.ItemsInAnyOrder( + m.In(t).Assert(json.RawMessage(decompressedBody2), m.JSONArray().Should(m.ItemsInAnyOrder( m.MapIncluding(m.KV("kind", m.Equal("index"))), m.MapIncluding(m.KV("kind", m.Equal("custom")), m.KV("key", m.Equal("eventkey2"))), ))) @@ -149,11 +159,17 @@ func TestSummarizingRelayPeriodicallyClosesInactiveEventProcessors(t *testing.T) assert.Equal(t, "tags1", request1a.Request.Header.Get(TagsHeader)) assert.Equal(t, "tags2", request2.Request.Header.Get(TagsHeader)) - m.In(t).Assert(json.RawMessage(request1a.Body), m.JSONArray().Should(m.ItemsInAnyOrder( + uncompressedBody1a, err := util.DecompressGzipData(request1a.Body) + assert.NoError(t, err) + + uncompressedBody2, err := util.DecompressGzipData(request2.Body) + assert.NoError(t, err) + + m.In(t).Assert(json.RawMessage(uncompressedBody1a), m.JSONArray().Should(m.ItemsInAnyOrder( m.MapIncluding(m.KV("kind", m.Equal("index"))), m.MapIncluding(m.KV("kind", m.Equal("custom")), m.KV("key", m.Equal("eventkey1a"))), ))) - m.In(t).Assert(json.RawMessage(request2.Body), m.JSONArray().Should(m.ItemsInAnyOrder( + m.In(t).Assert(json.RawMessage(uncompressedBody2), m.JSONArray().Should(m.ItemsInAnyOrder( m.MapIncluding(m.KV("kind", m.Equal("index"))), m.MapIncluding(m.KV("kind", m.Equal("custom")), m.KV("key", m.Equal("eventkey2"))), ))) @@ -165,8 +181,12 @@ func TestSummarizingRelayPeriodicallyClosesInactiveEventProcessors(t *testing.T) p.dispatcher.flush() request1b := expectSummarizedPayloadRequest(t, p.requestsCh) + + uncompressedBody1b, err := util.DecompressGzipData(request1b.Body) + assert.NoError(t, err) + assert.Equal(t, "tags1", request1b.Request.Header.Get(TagsHeader)) - m.In(t).Assert(json.RawMessage(request1b.Body), m.JSONArray().Should(m.ItemsInAnyOrder( + m.In(t).Assert(json.RawMessage(uncompressedBody1b), m.JSONArray().Should(m.ItemsInAnyOrder( m.MapIncluding(m.KV("kind", m.Equal("index"))), m.MapIncluding(m.KV("kind", m.Equal("custom")), m.KV("key", m.Equal("eventkey1b"))), ))) diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index 7543ba40..5ab89392 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -5,8 +5,12 @@ import ( "encoding/json" "errors" "net/http" + "strings" + + ct "github.com/launchdarkly/go-configtypes" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/util" "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/basictypes" @@ -213,3 +217,18 @@ func base64urlDecode(base64String string) ([]byte, error) { return idStr, nil } + +func GzipMiddleware(maxInboundPayloadSize ct.OptBase2Bytes) mux.MiddlewareFunc { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + isGzipped := strings.Contains(r.Header.Get("Content-Encoding"), "gzip") + reader, err := util.NewReader(r.Body, isGzipped, maxInboundPayloadSize) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + r.Body = reader + next.ServeHTTP(w, r) + }) + } +} diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 301ec5af..4bc81c75 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -345,7 +345,7 @@ func NewEnvContext( DataSource: dataSource, DataStore: storeAdapter, DiagnosticOptOut: !enableDiagnostics, - Events: ldcomponents.SendEvents(), + Events: ldcomponents.SendEvents().EnableGzip(true), HTTP: httpConfig.SDKHTTPConfigFactory, Logging: ldcomponents.Logging(). Loggers(envLoggers). diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index ed142779..bd4e1bcc 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/util" "github.com/launchdarkly/ld-relay/v8/internal/credential" @@ -355,7 +356,10 @@ func TestMetricsAreExportedForEnvironment(t *testing.T) { select { case req := <-requestsCh: mockLog.Loggers.Infof("received metrics events: %s", req.Body) - data := ldvalue.Parse(req.Body) + uncompressed, err := util.DecompressGzipData(req.Body) + require.NoError(t, err) + + data := ldvalue.Parse(uncompressed) event := data.GetByIndex(0) if !event.IsNull() { conns := event.GetByKey("connections") @@ -456,7 +460,10 @@ func TestEventDispatcherIsCreatedIfSendEventsIsTrueAndNotInOfflineMode(t *testin // Because the event schema version is >= 3, the event data should be forwarded verbatim with no processing. eventPost := helpers.RequireValue(t, requestsCh, time.Second) require.Equal(t, string(st.EnvMain.Config.SDKKey), eventPost.Request.Header.Get("Authorization")) - require.Equal(t, string(body), string(eventPost.Body)) + + decodedBody, err := util.DecompressGzipData(eventPost.Body) + require.NoError(t, err) + require.Equal(t, string(body), string(decodedBody)) }) } diff --git a/internal/util/reader.go b/internal/util/reader.go new file mode 100644 index 00000000..a80d6fdc --- /dev/null +++ b/internal/util/reader.go @@ -0,0 +1,111 @@ +package util + +import ( + "compress/gzip" + "errors" + "io" + + ct "github.com/launchdarkly/go-configtypes" +) + +// PayloadReader is an implementation of io.Reader that reads bytes off the request body +// optionally decompresses them, and has a limit attached. +// If the limit is reached, an error will be returned and the underlying stream will be closed. +// +// Note: limit is applied to *both* compressed and uncompressed number of bytes. This +// protects us from potential zipbombs +type PayloadReader struct { + IsGzipped bool + MaxBytes int64 + + uncompressedBytesRead int64 + + wrappedBaseStream *byteCountingReader + stream *io.LimitedReader +} + +// NewReader creates a new reader +func NewReader(r io.ReadCloser, isGzipped bool, maxInboundPayloadSize ct.OptBase2Bytes) (io.ReadCloser, error) { + // If this isn't compressed, and we don't want to limit the size, just + // return the original reader + if !isGzipped && !maxInboundPayloadSize.IsDefined() { + return r, nil + } + + var baseStream = &byteCountingReader{ + bytesRead: 0, + baseStream: r, + } + var s io.Reader = baseStream + + if isGzipped { + var err error + gzipReader, err := gzip.NewReader(s) + if err != nil { + return nil, err + } + + // If we don't want to limit the size, just return the gzip reader + if !maxInboundPayloadSize.IsDefined() { + return gzipReader, nil + } + + s = gzipReader + } + + maxBytes := int64(maxInboundPayloadSize.GetOrElse(0)) + stream := io.LimitReader(s, maxBytes).(*io.LimitedReader) + + payloadReader := &PayloadReader{ + IsGzipped: isGzipped, + MaxBytes: maxBytes, + wrappedBaseStream: baseStream, + stream: stream, + } + return payloadReader, nil +} + +// GetBytesRead returns the total number of bytes read off the original stream +func (pr *PayloadReader) GetBytesRead() int64 { + return pr.wrappedBaseStream.bytesRead +} + +// GetUncompressedBytesRead Total number of Bytes in the uncompressed stream. +// +// GetBytesRead and GetUncompressedBytesRead will return the same value if the stream is uncompressed. +func (pr *PayloadReader) GetUncompressedBytesRead() int64 { + return pr.uncompressedBytesRead +} + +func (pr *PayloadReader) Read(p []byte) (int, error) { + n, err := pr.stream.Read(p) + pr.uncompressedBytesRead += int64(n) + if err != nil { + return n, err + } + if pr.stream.N <= 0 { + _ = pr.Close() + return n, errors.New("max bytes exceeded") + } + return n, err +} + +func (pr *PayloadReader) Close() error { + c, ok := pr.wrappedBaseStream.baseStream.(io.Closer) + if ok { + return c.Close() + } + return nil +} + +// a simple reader decorator that keeps a running total of bytes +type byteCountingReader struct { + baseStream io.Reader + bytesRead int64 +} + +func (bt *byteCountingReader) Read(p []byte) (int, error) { + n, err := bt.baseStream.Read(p) + bt.bytesRead += int64(n) + return n, err +} diff --git a/internal/util/reader_test.go b/internal/util/reader_test.go new file mode 100644 index 00000000..b227ba37 --- /dev/null +++ b/internal/util/reader_test.go @@ -0,0 +1,72 @@ +package util + +import ( + "bytes" + "compress/gzip" + "io" + "strings" + "testing" + + "github.com/alecthomas/units" + ct "github.com/launchdarkly/go-configtypes" + "github.com/stretchr/testify/assert" +) + +func TestUncompressed(t *testing.T) { + // make sure the bytes read are correct for non-gzipped streams + nonZipBytes := []byte("this is a test") + + reader, _ := NewReader(io.NopCloser(bytes.NewReader(nonZipBytes)), false, ct.NewOptBase2Bytes(units.KiB)) + payloadReader, _ := reader.(*PayloadReader) + readBytes, _ := io.ReadAll(reader) + + assert.Equal(t, int64(len(nonZipBytes)), payloadReader.GetBytesRead()) + assert.Equal(t, int64(len(nonZipBytes)), payloadReader.GetUncompressedBytesRead()) + assert.Equal(t, nonZipBytes, readBytes) +} + +func TestPayloadBytesTracking(t *testing.T) { + ret := strings.Repeat("0", 1_000) + + // make sure the bytes read are correct for gzipped streams + nonZipBytes := []byte(ret) + + var b bytes.Buffer + w := gzip.NewWriter(&b) + w.Write(nonZipBytes) + w.Close() + zipBytes := b.Bytes() + + reader, _ := NewReader(io.NopCloser(bytes.NewReader(zipBytes)), true, ct.NewOptBase2Bytes(10*units.KiB)) + payloadReader, _ := reader.(*PayloadReader) + readBytes, _ := io.ReadAll(reader) + + assert.Equal(t, int64(len(zipBytes)), payloadReader.GetBytesRead()) // compressed is 32 + assert.Equal(t, int64(len(nonZipBytes)), payloadReader.GetUncompressedBytesRead()) // uncompressed is 1000 + assert.Equal(t, nonZipBytes, readBytes) +} + +func TestZipBombing(t *testing.T) { + ret := strings.Repeat("0", 1_000) + + nonZipBytes := []byte(ret) + + var b bytes.Buffer + w := gzip.NewWriter(&b) + w.Write(nonZipBytes) + w.Close() + zipBytes := b.Bytes() + + maxBytes := int64(100) + reader, _ := NewReader(io.NopCloser(bytes.NewReader(zipBytes)), true, ct.NewOptBase2Bytes(units.Base2Bytes(maxBytes))) + payloadReader, _ := reader.(*PayloadReader) + bytesRead, err := io.ReadAll(reader) + + // we should have an error because the uncompressed stream is larger than maxBytes + assert.Error(t, err) + assert.Equal(t, int64(len(bytesRead)), maxBytes) + + assert.Equal(t, int64(len(zipBytes)), payloadReader.GetBytesRead()) + assert.Equal(t, maxBytes, payloadReader.GetUncompressedBytesRead()) + +} diff --git a/internal/util/util.go b/internal/util/util.go index d8b41746..93dc697d 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -1,8 +1,11 @@ package util import ( + "bytes" + "compress/gzip" "encoding/json" "fmt" + "io" "net/url" ) @@ -36,3 +39,12 @@ func RedactURL(inputURL string) string { } return inputURL } + +func DecompressGzipData(data []byte) ([]byte, error) { + reader, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return data, err + } + + return io.ReadAll(reader) +} diff --git a/relay/autoconfig_key_change_test.go b/relay/autoconfig_key_change_test.go index 4ca0eb4a..36e709af 100644 --- a/relay/autoconfig_key_change_test.go +++ b/relay/autoconfig_key_change_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/util" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" @@ -73,7 +74,11 @@ func verifyEventSummarizingRelay(t *testing.T, p autoConfTestParams, url string, gotReq := helpers.RequireValue(t, p.eventRequestsCh, time.Second*5) assert.Equal(p.t, authKey.GetAuthorizationHeaderValue(), gotReq.Request.Header.Get("Authorization")) - eventsValue := ldvalue.Parse(gotReq.Body) + + decompressBody, err := util.DecompressGzipData(gotReq.Body) + assert.NoError(t, err) + + eventsValue := ldvalue.Parse(decompressBody) assert.Equal(p.t, "summary", eventsValue.GetByIndex(eventsValue.Count()-1).GetByKey("kind").StringValue()) } diff --git a/relay/endpoints_events_test.go b/relay/endpoints_events_test.go index 2b5ff29d..951e32a2 100644 --- a/relay/endpoints_events_test.go +++ b/relay/endpoints_events_test.go @@ -14,6 +14,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/browser" "github.com/launchdarkly/ld-relay/v8/internal/events" st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/util" ct "github.com/launchdarkly/go-configtypes" helpers "github.com/launchdarkly/go-test-helpers/v3" @@ -35,7 +36,9 @@ type relayEventsTestParams struct { func (p relayEventsTestParams) requirePublishedEvent(t *testing.T, data []byte) publishedEvent { event := helpers.RequireValue(t, p.publishedEvents, time.Second*3) - assert.JSONEq(t, string(data), string(event.data)) + uncompressed, err := util.DecompressGzipData(event.data) + assert.NoError(t, err) + assert.JSONEq(t, string(data), string(uncompressed)) return event } diff --git a/relay/relay_routes.go b/relay/relay_routes.go index 5484e0c7..9ac4555f 100644 --- a/relay/relay_routes.go +++ b/relay/relay_routes.go @@ -124,18 +124,18 @@ func (r *Relay) makeRouter() *mux.Router { clientSideStreamEvalRouter.Handle("", middleware.CountBrowserConns(jsPingWithUser)).Methods("REPORT", "OPTIONS") mobileEventsRouter := router.PathPrefix("/mobile").Subrouter() - mobileEventsRouter.Use(mobileMiddlewareStack) + mobileEventsRouter.Use(mobileMiddlewareStack, middleware.GzipMiddleware(r.config.Events.MaxInboundPayloadSize)) mobileEventsRouter.Handle("/events/bulk", bulkEventHandler(basictypes.MobileSDK, ldevents.AnalyticsEventDataKind, offlineMode)).Methods("POST") mobileEventsRouter.Handle("/events", bulkEventHandler(basictypes.MobileSDK, ldevents.AnalyticsEventDataKind, offlineMode)).Methods("POST") mobileEventsRouter.Handle("", bulkEventHandler(basictypes.MobileSDK, ldevents.AnalyticsEventDataKind, offlineMode)).Methods("POST") mobileEventsRouter.Handle("/events/diagnostic", bulkEventHandler(basictypes.MobileSDK, ldevents.DiagnosticEventDataKind, offlineMode)).Methods("POST") clientSideBulkEventsRouter := router.PathPrefix("/events/bulk/{envId}").Subrouter() - clientSideBulkEventsRouter.Use(jsClientSideMiddlewareStack(clientSideBulkEventsRouter)) + clientSideBulkEventsRouter.Use(jsClientSideMiddlewareStack(clientSideBulkEventsRouter), middleware.GzipMiddleware(r.config.Events.MaxInboundPayloadSize)) clientSideBulkEventsRouter.Handle("", bulkEventHandler(basictypes.JSClientSDK, ldevents.AnalyticsEventDataKind, offlineMode)).Methods("POST", "OPTIONS") clientSideDiagnosticEventsRouter := router.PathPrefix("/events/diagnostic/{envId}").Subrouter() - clientSideDiagnosticEventsRouter.Use(jsClientSideMiddlewareStack(clientSideBulkEventsRouter)) + clientSideDiagnosticEventsRouter.Use(jsClientSideMiddlewareStack(clientSideBulkEventsRouter), middleware.GzipMiddleware(r.config.Events.MaxInboundPayloadSize)) clientSideDiagnosticEventsRouter.Handle("", bulkEventHandler(basictypes.JSClientSDK, ldevents.DiagnosticEventDataKind, offlineMode)).Methods("POST", "OPTIONS") clientSideImageEventsRouter := router.PathPrefix("/a/{envId}.gif").Subrouter() @@ -144,8 +144,12 @@ func (r *Relay) makeRouter() *mux.Router { serverSideRouter := router.PathPrefix("").Subrouter() serverSideRouter.Use(serverSideMiddlewareStack) - serverSideRouter.Handle("/bulk", bulkEventHandler(basictypes.ServerSDK, ldevents.AnalyticsEventDataKind, offlineMode)).Methods("POST") - serverSideRouter.Handle("/diagnostic", bulkEventHandler(basictypes.ServerSDK, ldevents.DiagnosticEventDataKind, offlineMode)).Methods("POST") + + serverSideBulkEventsRouter := serverSideRouter.NewRoute().Subrouter() + serverSideBulkEventsRouter.Use(middleware.GzipMiddleware(r.config.Events.MaxInboundPayloadSize)) + serverSideBulkEventsRouter.Handle("/bulk", bulkEventHandler(basictypes.ServerSDK, ldevents.AnalyticsEventDataKind, offlineMode)).Methods("POST") + serverSideBulkEventsRouter.Handle("/diagnostic", bulkEventHandler(basictypes.ServerSDK, ldevents.DiagnosticEventDataKind, offlineMode)).Methods("POST") + serverSideRouter.Handle("/all", middleware.CountServerConns(middleware.Streaming( streamHandler(r.serverSideStreamProvider, serverSideStreamLogMessage), ))).Methods("GET")