[loadgen] Separate into otelbench for benchmarking, otelsoak for soak testing

.github/workflows/checks.yml

@@ -227,5 +227,5 @@ jobs:
       - uses: actions/checkout@v4
       - name: Build testing component's collector and validate example configuration
         run: make elasticcol-validate
-      - name: Validate loadgen example config
-        run: make loadgen-validate
+      - name: Validate otelsoak example config
+        run: make otelsoak-validate

Makefile

@@ -91,12 +91,12 @@ builddocker:
 	fi; \
 	docker build -t $$IMAGE_NAME -f distributions/elastic-components/Dockerfile .
 
-# Validate that the Elastic components collector can run with the example loadgen configuration.
-.PHONY: loadgencol-validate
-loadgen-validate: genelasticcol
-	ELASTIC_APM_SERVER_URL=http://localhost:8200 ELASTIC_APM_API_KEY=foobar ./_build/elastic-collector-components validate --config ./loadgen/config.example.yaml
-
-# Run loadgen
-.PHONY: loadgencol-run
-loadgen-run: genelasticcol
-	TESTDATA_DIR=./loadgen ./_build/elastic-collector-components --config ./loadgen/config.example.yaml $(ARGS)
+# Validate that the Elastic components collector can run with the example otelsoak configuration.
+.PHONY: otelsoak-validate
+otelsoak-validate: genelasticcol
+	ELASTIC_APM_SERVER_URL=http://localhost:8200 ELASTIC_APM_API_KEY=foobar ./loadgen/cmd/otelsoak/otelsoak validate --config ./loadgen/cmd/otelsoak/config.example.yaml
+
+# Run otelsoak
+.PHONY: otelsoak-run
+otelsoak-run: genelasticcol
+	./loadgen/cmd/otelsoak/otelsoak --config ./loadgen/cmd/otelsoak/config.example.yaml $(ARGS)

loadgen/README.md

@@ -1,19 +1,11 @@
-# loadgen: the load generator based on OTel collector
+# loadgen: OTel load generation tooling
 
-To generate load to an OTLP target, run Elastic collector components distro with specific pipelines to replay canned data at a configurable rate.
+In `cmd/` directory, there are
+- [otelsoak](./cmd/otelsoak/README.md)
+    - Load generator that is exactly an OTel collector. It sends load to an endpoint and never terminates.
+    - Suitable for soak testing
+- [otelbench](./cmd/otelbench/README.md)
+    - Load generator based on OTel collector and Go benchmarking. It sends load to an endpoint, terminates after a configured duration, and outputs statistics.
+    - Suitable for benchmarking
 
-See an example configuration at `config.example.yaml`. There are rate limiting and trace ID rewriting by default.
-
-## Usage
-
-```
-../_build/elastic-collector-components --config config.example.yaml --set "exporter.otlp.endpoint=http://localhost:8200" --set "exporter.otlp.headers.Authorization=ApiKey xxx" --set "exporter.otlp.headers.X-FOO-HEADER=bar"
-```
-
-Alternatively, there's `ELASTIC_APM_SERVER_URL` and `ELASTIC_APM_API_KEY` env var handling out of the box. `ELASTIC_APM_SECRET_TOKEN` is NOT supported without changing `config.example.yaml`.
-
-```
-ELASTIC_APM_SERVER_URL=http://localhost:8200 ELASTIC_APM_API_KEY=some_api_key ../_build/elastic-collector-components --config config.example.yaml
-```
-
-Even better, create your own `config.yaml` from `config.example.yaml` to fit your needs.
+otelsoak and otelbench are synonymous to apmsoak and apmbench in [elastic/apm-perf](https://github.com/elastic/apm-perf).

loadgen/cmd/otelbench/Makefile

@@ -0,0 +1 @@
+include ../../../Makefile.Common

loadgen/cmd/otelbench/README.md

@@ -0,0 +1,129 @@
+# otelbench
+
+otelbench wraps the collector inside a Go test benchmark. It outputs throughput numbers in standard Go benchmark format. It uses a collector config yaml as a base, and applies some overrides from command line options.
+
+## Usage
+
+```
+Usage of ./otelbench:
+  -api-key string
+        API key for target server
+  -config string
+        path to collector config yaml (default "config.yaml")
+  -endpoint value
+        target server endpoint for both otlp and otlphttp exporters (default to value in config yaml), equivalent to setting both -endpoint-otlp and -endpoint-otlphttp
+  -endpoint-otlp value
+        target server endpoint for otlp exporter (default to value in config yaml)
+  -endpoint-otlphttp value
+        target server endpoint for otlphttp exporter (default to value in config yaml)
+  -exporter-otlp
+        benchmark exporter otlp (default true)
+  -exporter-otlphttp
+        benchmark exporter otlphttp (default true)
+  -header value
+        extra headers in key=value format when sending data to the server. Can be repeated. e.g. -header X-FIRST-HEADER=foo -header X-SECOND-HEADER=bar
+  -insecure
+        disable TLS, ignored by otlphttp exporter (default to value in config yaml)
+  -insecure-skip-verify
+        skip validating the remote server TLS certificates (default to value in config yaml)
+  -logs
+        benchmark logs (default true)
+  -metrics
+        benchmark metrics (default true)
+  -secret-token string
+        secret token for target server
+  -test.bench regexp
+        run only benchmarks matching regexp
+  -test.benchmem
+        print memory allocations for benchmarks
+  -test.benchtime d
+        run each benchmark for duration d or N times if `d` is of the form Nx (default 1s)
+  -test.blockprofile file
+        write a goroutine blocking profile to file
+  -test.blockprofilerate rate
+        set blocking profile rate (see runtime.SetBlockProfileRate) (default 1)
+  -test.count n
+        run tests and benchmarks n times (default 1)
+  -test.coverprofile file
+        write a coverage profile to file
+  -test.cpu list
+        comma-separated list of cpu counts to run each test with
+  -test.cpuprofile file
+        write a cpu profile to file
+  -test.failfast
+        do not start new tests after the first test failure
+  -test.fullpath
+        show full file names in error messages
+  -test.fuzz regexp
+        run the fuzz test matching regexp
+  -test.fuzzcachedir string
+        directory where interesting fuzzing inputs are stored (for use only by cmd/go)
+  -test.fuzzminimizetime value
+        time to spend minimizing a value after finding a failing input (default 1m0s)
+  -test.fuzztime value
+        time to spend fuzzing; default is to run indefinitely
+  -test.fuzzworker
+        coordinate with the parent process to fuzz random values (for use only by cmd/go)
+  -test.gocoverdir string
+        write coverage intermediate files to this directory
+  -test.list regexp
+        list tests, examples, and benchmarks matching regexp then exit
+  -test.memprofile file
+        write an allocation profile to file
+  -test.memprofilerate rate
+        set memory allocation profiling rate (see runtime.MemProfileRate)
+  -test.mutexprofile string
+        write a mutex contention profile to the named file after execution
+  -test.mutexprofilefraction int
+        if >= 0, calls runtime.SetMutexProfileFraction() (default 1)
+  -test.outputdir dir
+        write profiles to dir
+  -test.paniconexit0
+        panic on call to os.Exit(0)
+  -test.parallel n
+        run at most n tests in parallel (default 16)
+  -test.run regexp
+        run only tests and examples matching regexp
+  -test.short
+        run smaller test suite to save time
+  -test.shuffle string
+        randomize the execution order of tests and benchmarks (default "off")
+  -test.skip regexp
+        do not list or run tests matching regexp
+  -test.testlogfile file
+        write test action log to file (for use only by cmd/go)
+  -test.timeout d
+        panic test binary after duration d (default 0, timeout disabled)
+  -test.trace file
+        write an execution trace to file
+  -test.v
+        verbose: print additional output
+  -traces
+        benchmark traces (default true)
+```
+
+## Example usage
+
+To send to a local apm-server
+
+```shell
+./otelbench -config=./config.yaml -endpoint=http://localhost:8200 -secret-token=foobar -insecure
+```
+
+To send to an ESS apm-server
+
+```shell
+./otelbench -test.benchtime=1m -config=./config.yaml -endpoint=https://foobar.apm.europe-west2.gcp.elastic-cloud.com:443 -api-key=some_api_key
+```
+
+To send to an OTel collector with a special otlphttp path
+
+```shell
+./otelbench -config=./config.yaml -endpoint-otlp=localhost:4317 -endpoint-otlphttp=https://localhost:4318/prefix -api-key some_api_key
+```
+
+It is possible to run with a customized config to avoid passing in command line options every time
+
+```shell
+./otelbench -config=./my-custom-config.yaml
+```

loadgen/cmd/otelbench/collector.go

@@ -0,0 +1,91 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"context"
+	"os"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/confmap"
+	"go.opentelemetry.io/collector/confmap/provider/envprovider"
+	"go.opentelemetry.io/collector/confmap/provider/fileprovider"
+	"go.opentelemetry.io/collector/confmap/provider/httpprovider"
+	"go.opentelemetry.io/collector/confmap/provider/httpsprovider"
+	"go.opentelemetry.io/collector/confmap/provider/yamlprovider"
+	"go.opentelemetry.io/collector/otelcol"
+
+	"github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver"
+)
+
+const (
+	buildDescription = "otelbench distribution"
+	buildVersion     = "0.0.1"
+)
+
+func RunCollector(ctx context.Context, stop chan struct{}, configFiles []string, logsDone, metricsDone, tracesDone chan loadgenreceiver.Stats) error {
+	settings, err := NewCollectorSettings(configFiles, logsDone, metricsDone, tracesDone)
+	if err != nil {
+		return err
+	}
+
+	svc, err := otelcol.NewCollector(settings)
+	if err != nil {
+		return err
+	}
+
+	// cancel context on stop from event manager
+	cancelCtx, cancel := context.WithCancel(ctx)
+	go func() {
+		<-stop
+		cancel()
+	}()
+	defer cancel()
+
+	return svc.Run(cancelCtx)
+}
+
+func NewCollectorSettings(configPaths []string, logsDone, metricsDone, tracesDone chan loadgenreceiver.Stats) (otelcol.CollectorSettings, error) {
+	buildInfo := component.BuildInfo{
+		Command:     os.Args[0],
+		Description: buildDescription,
+		Version:     buildVersion,
+	}
+	configProviderSettings := otelcol.ConfigProviderSettings{
+		ResolverSettings: confmap.ResolverSettings{
+			URIs: configPaths,
+			ProviderFactories: []confmap.ProviderFactory{
+				fileprovider.NewFactory(),
+				envprovider.NewFactory(),
+				yamlprovider.NewFactory(),
+				httpprovider.NewFactory(),
+				httpsprovider.NewFactory(),
+			},
+			ConverterFactories: []confmap.ConverterFactory{},
+		},
+	}
+
+	return otelcol.CollectorSettings{
+		Factories:              func() (otelcol.Factories, error) { return components(logsDone, metricsDone, tracesDone) },
+		BuildInfo:              buildInfo,
+		ConfigProviderSettings: configProviderSettings,
+		// we're handling DisableGracefulShutdown via the cancelCtx being passed
+		// to the collector's Run method in the Run function
+		DisableGracefulShutdown: true,
+	}, nil
+}

loadgen/cmd/otelbench/components.go

@@ -0,0 +1,82 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor"
+	"go.opentelemetry.io/collector/connector"
+	"go.opentelemetry.io/collector/exporter"
+	"go.opentelemetry.io/collector/exporter/debugexporter"
+	"go.opentelemetry.io/collector/exporter/nopexporter"
+	"go.opentelemetry.io/collector/exporter/otlpexporter"
+	"go.opentelemetry.io/collector/exporter/otlphttpexporter"
+	"go.opentelemetry.io/collector/extension"
+	"go.opentelemetry.io/collector/otelcol"
+	"go.opentelemetry.io/collector/processor"
+	"go.opentelemetry.io/collector/receiver"
+	"go.opentelemetry.io/collector/receiver/nopreceiver"
+
+	"github.com/elastic/opentelemetry-collector-components/processor/ratelimitprocessor"
+	"github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver"
+)
+
+func components(logsDone, metricsDone, tracesDone chan loadgenreceiver.Stats) (otelcol.Factories, error) {
+	var err error
+	factories := otelcol.Factories{}
+
+	// Receivers
+	factories.Receivers, err = receiver.MakeFactoryMap(
+		loadgenreceiver.NewFactoryWithDone(logsDone, metricsDone, tracesDone),
+		nopreceiver.NewFactory(),
+	)
+	if err != nil {
+		return otelcol.Factories{}, err
+	}
+
+	// Processors
+	factories.Processors, err = processor.MakeFactoryMap(
+		ratelimitprocessor.NewFactory(),
+		transformprocessor.NewFactory(),
+	)
+	if err != nil {
+		return otelcol.Factories{}, err
+	}
+
+	// Exporters
+	factories.Exporters, err = exporter.MakeFactoryMap(
+		otlpexporter.NewFactory(),
+		otlphttpexporter.NewFactory(),
+		debugexporter.NewFactory(),
+		nopexporter.NewFactory(),
+	)
+	if err != nil {
+		return otelcol.Factories{}, err
+	}
+
+	factories.Connectors, err = connector.MakeFactoryMap()
+	if err != nil {
+		return otelcol.Factories{}, err
+	}
+
+	factories.Extensions, err = extension.MakeFactoryMap()
+	if err != nil {
+		return otelcol.Factories{}, err
+	}
+
+	return factories, err
+}