Add loadgenreceiver and loadgen config (#246)

Add a loadgen config that configures elastic collector components distro to replay and rewrite canned opentelemetry-demo data at a configurable rate. It uses a new loadgenreceiver and the existing ratelimitprocessor.

.github/dependabot.yml

@@ -77,6 +77,21 @@ updates:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
 
+  - package-ecosystem: "gomod"
+    directory: "/receiver/loadgenreceiver"
+    schedule:
+      interval: "daily"
+    labels:
+      - automation
+    groups:
+      otel-dependencies:
+        patterns:
+          - "go.opentelemetry.io/*"
+          - "github.com/open-telemetry/opentelemetry-collector-contrib/*"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+
   - package-ecosystem: "gomod"
     directory: "/internal/tools/"
     schedule:

.github/workflows/checks.yml

@@ -227,3 +227,5 @@ jobs:
       - uses: actions/checkout@v4
       - name: Build testing component's collector and validate example configuration
         run: make elasticcol-validate
+      - name: Validate loadgen example config
+        run: make loadgen-validate

Makefile

@@ -89,4 +89,14 @@ builddocker:
 	else \
 		IMAGE_NAME=elastic-collector-components:$(TAG); \
 	fi; \
-	docker build -t $$IMAGE_NAME -f distributions/elastic-components/Dockerfile .
+	docker build -t $$IMAGE_NAME -f distributions/elastic-components/Dockerfile .
+
+# Validate that the Elastic components collector can run with the example loadgen configuration.
+.PHONY: loadgencol-validate
+loadgen-validate: genelasticcol
+	ELASTIC_APM_SERVER_URL=http://localhost:8200 ELASTIC_APM_API_KEY=foobar ./_build/elastic-collector-components validate --config ./loadgen/config.example.yaml
+
+# Run loadgen
+.PHONY: loadgencol-run
+loadgen-run: genelasticcol
+	TESTDATA_DIR=./loadgen ./_build/elastic-collector-components --config ./loadgen/config.example.yaml $(ARGS)

README.md

@@ -33,3 +33,7 @@ In order to build a collector with a custom component, e.g. for testing purposes
    make builddocker TAG=v0.1.0 USERNAME=johndoe
    ``` 
 
+## Load generator
+
+See [./loadgen/README.md](./loadgen/README.md).
+

distributions/elastic-components/manifest.yaml

@@ -28,6 +28,7 @@ receivers:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sobjectsreceiver v0.116.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.116.0
   - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.116.0
+  - gomod: github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver v0.0.0
 
 processors:
   - gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.116.0
@@ -41,6 +42,8 @@ processors:
   - gomod: github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.13.0
   - gomod: github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.3.0
   - gomod: github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor v0.3.0
+  - gomod: github.com/elastic/opentelemetry-collector-components/processor/ratelimitprocessor v0.0.0
+    path: ./processor/ratelimitprocessor
 
 exporters:
   - gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.116.0
@@ -63,3 +66,5 @@ replaces:
   - github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor => ../processor/elastictraceprocessor
   - github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor => ../processor/lsmintervalprocessor
   - github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector => ../connector/signaltometricsconnector
+  - github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver => ../receiver/loadgenreceiver
+  - github.com/elastic/opentelemetry-collector-components/processor/ratelimitprocessor => ../processor/ratelimitprocessor

loadgen/README.md

@@ -0,0 +1,19 @@
+# loadgen: the load generator based on OTel collector
+
+To generate load to an OTLP target, run Elastic collector components distro with specific pipelines to replay canned data at a configurable rate.
+
+See an example configuration at `config.example.yaml`. There are rate limiting and trace ID rewriting by default.
+
+## Usage
+
+```
+../_build/elastic-collector-components --config config.example.yaml --set "exporter.otlp.endpoint=http://localhost:8200" --set "exporter.otlp.headers.Authorization=ApiKey xxx" --set "exporter.otlp.headers.X-FOO-HEADER=bar"
+```
+
+Alternatively, there's `ELASTIC_APM_SERVER_URL` and `ELASTIC_APM_API_KEY` env var handling out of the box. `ELASTIC_APM_SECRET_TOKEN` is NOT supported without changing `config.example.yaml`.
+
+```
+ELASTIC_APM_SERVER_URL=http://localhost:8200 ELASTIC_APM_API_KEY=some_api_key ../_build/elastic-collector-components --config config.example.yaml
+```
+
+Even better, create your own `config.yaml` from `config.example.yaml` to fit your needs.

loadgen/config.example.yaml

@@ -0,0 +1,48 @@
+receivers:
+  loadgen:
+
+exporters:
+  otlp:
+    endpoint: "${env:ELASTIC_APM_SERVER_URL}"
+    headers:
+      Authorization: "ApiKey ${env:ELASTIC_APM_API_KEY}"
+#      Authorization: "Bearer ${env:ELASTIC_APM_SECRET_TOKEN}"
+    sending_queue:
+      enabled: false
+    timeout: 60s
+  debug:
+
+
+processors:
+  transform/rewrite:  # Rewrite telemetry to increase cardinality
+    trace_statements:
+      - context: span
+        statements:
+          # The worst way to generate a random ID, but is the simplest in OTTL
+          # Only randomize trace ID such that span relationships are still maintained
+          - set(trace_id.string, Substring(MD5(UUID()), 0, 32))
+
+  ratelimit:
+    strategy: records
+    rate: 5000
+    burst: 5000
+    throttle_behavior: delay
+
+
+service:
+  pipelines:
+    logs:
+      receivers: [loadgen]
+      processors: [ratelimit, transform/rewrite]
+      exporters: [otlp, debug]
+    metrics:
+      receivers: [loadgen]
+      processors: [ratelimit, transform/rewrite]
+      exporters: [otlp, debug]
+    traces:
+      receivers: [loadgen]
+      processors: [ratelimit, transform/rewrite]
+      exporters: [otlp, debug]
+#  telemetry:
+#    logs:
+#      level: debug

receiver/loadgenreceiver/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

receiver/loadgenreceiver/README.md

@@ -0,0 +1,56 @@
+# Loadgen receiver
+
+<!-- status autogenerated section -->
+| Status        |           |
+| ------------- |-----------|
+| Stability     | [development]: metrics, logs, traces   |
+| Distributions | [] |
+| Issues        | [![Open issues](https://img.shields.io/github/issues-search/elastic/opentelemetry-collector-components?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Floadgen%20&label=open&color=orange&logo=opentelemetry)](https://github.com/elastic/opentelemetry-collector-components/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Floadgen) [![Closed issues](https://img.shields.io/github/issues-search/elastic/opentelemetry-collector-components?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Floadgen%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/elastic/opentelemetry-collector-components/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Floadgen) |
+
+[development]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#development
+<!-- end autogenerated section -->
+
+The Loadgen receiver is a specialized OpenTelemetry receiver designed to generate metrics, logs, and traces for testing, benchmarking, and validation purposes.
+
+## How Telemetry Is Generated
+
+The receiver utilizes pre-captured telemetry data from a typical OpenTelemetry Demo execution, which represents a diverse set of application scenarios and workloads. When generating telemetry, the receiver uses this base data, dynamically adjusting timestamps to match the current time. This approach ensures the generated telemetry retains the structure and content of authentic OpenTelemetry Demo data while appearing as though it were generated in real-time. The pre-captured telemetry is embedded in the receiver and can be found in [testdata](./testdata/) directory.
+
+## Rate limiting
+
+The receiver generates telemetry as quickly as possible. Any rate limiting should be done via backpressure using [processor/ratelimitprocessor](/processor/ratelimitprocessor).
+
+## Telemetry cardinality
+
+The receiver only rewrites timestamps to Now, and does not modify any other fields. Therefore, it will have the same cardinality as the original canned data. To simulate higher cardinality (e.g. trace ID, service name), use `transform` processor with OTTL to rewrite fields. 
+
+## Sample configuration
+
+```yaml
+receivers:
+  loadgen:
+
+processors:
+
+exporters:
+  debug:
+    verbosity: detailed
+
+service:
+  telemetry:
+    logs:
+      level: debug
+  pipelines:
+    metrics:
+      receivers: [loadgen]
+      processors:
+      exporters: [debug]
+    logs:
+      receivers: [loadgen]
+      processors:
+      exporters: [debug]
+    traces:
+      receivers: [loadgen]
+      processors:
+      exporters: [debug]
+```

receiver/loadgenreceiver/config.go

@@ -0,0 +1,58 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package loadgenreceiver // import "github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver"
+
+import (
+	"go.opentelemetry.io/collector/component"
+)
+
+type (
+	JsonlFile string
+)
+
+// Config defines configuration for HostMetrics receiver.
+type Config struct {
+	Metrics MetricsConfig `mapstructure:"metrics"`
+	Logs    LogsConfig    `mapstructure:"logs"`
+	Traces  TracesConfig  `mapstructure:"traces"`
+}
+
+type MetricsConfig struct {
+	// JsonlFile is an optional configuration option to specify the path to
+	// get the base generated signals from.
+	JsonlFile `mapstructure:"jsonl_file"`
+}
+
+type LogsConfig struct {
+	// JsonlFile is an optional configuration option to specify the path to
+	// get the base generated signals from.
+	JsonlFile `mapstructure:"jsonl_file"`
+}
+
+type TracesConfig struct {
+	// JsonlFile is an optional configuration option to specify the path to
+	// get the base generated signals from.
+	JsonlFile `mapstructure:"jsonl_file"`
+}
+
+var _ component.Config = (*Config)(nil)
+
+// Validate checks the receiver configuration is valid
+func (cfg *Config) Validate() error {
+	return nil
+}

receiver/loadgenreceiver/doc.go

@@ -0,0 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:generate mdatagen metadata.yaml
+
+package loadgenreceiver // import "github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver"

receiver/loadgenreceiver/factory.go

@@ -0,0 +1,39 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package loadgenreceiver // import "github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver"
+
+import (
+	"github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver/internal/metadata"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/receiver"
+)
+
+// CreateDefaultConfig creates the default configuration for the Scraper.
+func createDefaultReceiverConfig() component.Config {
+	return &Config{}
+}
+
+func NewFactory() receiver.Factory {
+	return receiver.NewFactory(
+		metadata.Type,
+		createDefaultReceiverConfig,
+		receiver.WithMetrics(createMetricsReceiver, component.StabilityLevelDevelopment),
+		receiver.WithTraces(createTracesReceiver, component.StabilityLevelDevelopment),
+		receiver.WithLogs(createLogsReceiver, component.StabilityLevelDevelopment),
+	)
+}