[jaeger-v2] Add support for Elasticsearch

cmd/jaeger/es_config.yaml

@@ -0,0 +1,34 @@
+service:
+  extensions: [jaeger_storage, jaeger_query]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [jaeger_storage_exporter]
+
+extensions:
+  jaeger_query:
+    trace_storage: es_main
+    trace_storage_archive: es_archive
+    ui_config: ./cmd/jaeger/config-ui.json
+
+  jaeger_storage:
+    elasticsearch:
+      es_main:
+        server_urls: http://127.0.0.1:9200
+        log_level: "error"
+      es_archive:
+        server_urls: http://127.0.0.1:9200
+        log_level: "error"
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+processors:
+  batch:
+
+exporters:
+  jaeger_storage_exporter:
+    trace_storage: es_main

cmd/jaeger/internal/extension/jaegerstorage/config.go

@@ -7,14 +7,16 @@ import (
 	"fmt"
 	"reflect"
 
+	esCfg "github.com/jaegertracing/jaeger/pkg/es/config"
 	memoryCfg "github.com/jaegertracing/jaeger/pkg/memory/config"
 	badgerCfg "github.com/jaegertracing/jaeger/plugin/storage/badger"
 )
 
 // Config has the configuration for jaeger-query,
 type Config struct {
-	Memory map[string]memoryCfg.Configuration   `mapstructure:"memory"`
-	Badger map[string]badgerCfg.NamespaceConfig `mapstructure:"badger"`
+	Memory        map[string]memoryCfg.Configuration   `mapstructure:"memory"`
+	Badger        map[string]badgerCfg.NamespaceConfig `mapstructure:"badger"`
+	Elasticsearch map[string]esCfg.Configuration       `mapstructure:"elasticsearch"`
 	// TODO add other storage types here
 	// TODO how will this work with 3rd party storage implementations?
 	//      Option: instead of looking for specific name, check interface.

cmd/jaeger/internal/extension/jaegerstorage/extension.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/jaegertracing/jaeger/pkg/metrics"
 	"github.com/jaegertracing/jaeger/plugin/storage/badger"
+	"github.com/jaegertracing/jaeger/plugin/storage/es"
 	"github.com/jaegertracing/jaeger/plugin/storage/memory"
 	"github.com/jaegertracing/jaeger/storage"
 )
@@ -88,6 +89,22 @@ func (s *storageExt) Start(ctx context.Context, host component.Host) error {
 		s.factories[name] = factory
 	}
 
+	for name, e := range s.config.Elasticsearch {
+		if _, ok := s.factories[name]; ok {
+			return fmt.Errorf("duplicate elasticsearch storage name %s", name)
+		}
+		var err error
+		factory, err := es.NewFactoryWithConfig(
+			e,
+			metrics.NullFactory,
+			s.logger.With(zap.String("storage_name", name)),
+		)
+		if err != nil {
+			return fmt.Errorf("failed to initialize elasticsearch storage: %w", err)
+		}
+		s.factories[name] = factory
+
+	}
 	// TODO add support for other backends
 	return nil
 }

cmd/jaeger/internal/extension/jaegerstorage/extension_test.go

@@ -6,6 +6,8 @@ package jaegerstorage
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -16,9 +18,11 @@ import (
 	nooptrace "go.opentelemetry.io/otel/trace/noop"
 	"go.uber.org/zap"
 
+	esCfg "github.com/jaegertracing/jaeger/pkg/es/config"
 	memoryCfg "github.com/jaegertracing/jaeger/pkg/memory/config"
 	"github.com/jaegertracing/jaeger/pkg/metrics"
 	badgerCfg "github.com/jaegertracing/jaeger/plugin/storage/badger"
+	"github.com/jaegertracing/jaeger/plugin/storage/es"
 	"github.com/jaegertracing/jaeger/storage"
 	"github.com/jaegertracing/jaeger/storage/dependencystore"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
@@ -27,6 +31,7 @@ import (
 const (
 	memstoreName = "memstore"
 	badgerName   = "badgerstore"
+	esName       = "esstore"
 )
 
 type storageHost struct {
@@ -188,6 +193,74 @@ func TestBadgerStorageExtensionError(t *testing.T) {
 	require.EqualError(t, err, "failed to initialize badger storage: Error Creating Dir: \"\" error: mkdir : no such file or directory")
 }
 
+func TestESStorageExtension(t *testing.T) {
+	ctx := context.Background()
+	telemetrySettings := component.TelemetrySettings{
+		Logger:         zap.NewNop(),
+		TracerProvider: nooptrace.NewTracerProvider(),
+		MeterProvider:  noopmetric.NewMeterProvider(),
+	}
+
+	mockEsServerResponse := []byte(`
+	{
+		"Version": {
+			"Number": "6"
+		}
+	}
+	`)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(mockEsServerResponse)
+	}))
+
+	defer server.Close()
+	config := &Config{
+		Elasticsearch: map[string]esCfg.Configuration{
+			esName: {
+				Servers:  []string{server.URL},
+				LogLevel: "error",
+			},
+		},
+	}
+
+	require.NoError(t, config.Validate())
+
+	extensionFactory := NewFactory()
+	storageExtension, err := extensionFactory.CreateExtension(ctx, extension.CreateSettings{
+		ID:                ID,
+		TelemetrySettings: telemetrySettings,
+		BuildInfo:         component.NewDefaultBuildInfo(),
+	}, config)
+	require.NoError(t, err)
+
+	host := componenttest.NewNopHost()
+	err = storageExtension.Start(ctx, host)
+	require.NoError(t, err)
+
+	err = storageExtension.Start(ctx, host)
+	t.Cleanup(func() { require.NoError(t, storageExtension.Shutdown(ctx)) })
+	require.Error(t, err)
+	require.EqualError(t, err, fmt.Sprintf("duplicate elasticsearch storage name %s", esName))
+}
+
+func TestESStorageExtensionError(t *testing.T) {
+	ctx := context.Background()
+	factory, _ := es.NewFactoryWithConfig(esCfg.Configuration{}, metrics.NullFactory, zap.NewNop())
+	ext := storageExt{
+		config: &Config{
+			Elasticsearch: map[string]esCfg.Configuration{
+				esName: {},
+			},
+		},
+		logger:    zap.NewNop(),
+		factories: map[string]storage.Factory{"elasticsearch": factory},
+	}
+	err := ext.Start(ctx, componenttest.NewNopHost())
+	require.Error(t, err)
+	fmt.Println(err)
+	require.EqualError(t, err, "failed to initialize elasticsearch storage: failed to create primary Elasticsearch client: no servers specified")
+}
+
 func makeStorageExtension(t *testing.T, memstoreName string) component.Component {
 	extensionFactory := NewFactory()
 

plugin/storage/es/factory.go

@@ -83,6 +83,23 @@ func NewFactory() *Factory {
 	}
 }
 
+func NewFactoryWithConfig(
+	cfg config.Configuration,
+	metricsFactory metrics.Factory,
+	logger *zap.Logger,
+) (*Factory, error) {
+	f := NewFactory()
+	f.InitFromOptions(Options{
+		Primary: namespaceConfig{Configuration: cfg},
+		others:  make(map[string]*namespaceConfig),
+	})
+	err := f.Initialize(metricsFactory, logger)
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
 // AddFlags implements plugin.Configurable
 func (f *Factory) AddFlags(flagSet *flag.FlagSet) {
 	f.Options.AddFlags(flagSet)

plugin/storage/es/factory_test.go

@@ -263,6 +263,26 @@ func TestInitFromOptions(t *testing.T) {
 	assert.Equal(t, o.Get(archiveNamespace), f.archiveConfig)
 }
 
+func TestESStorageFactoryWithConfig(t *testing.T) {
+	cfg := escfg.Configuration{}
+	_, err := NewFactoryWithConfig(cfg, metrics.NullFactory, zap.NewNop())
+	require.Error(t, err)
+	require.ErrorContains(t, err, "failed to create primary Elasticsearch client")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(mockEsServerResponse)
+	}))
+	defer server.Close()
+
+	cfg = escfg.Configuration{
+		Servers:  []string{server.URL},
+		LogLevel: "error",
+	}
+	factory, err := NewFactoryWithConfig(cfg, metrics.NullFactory, zap.NewNop())
+	require.NoError(t, err)
+	defer factory.Close()
+}
+
 func TestPasswordFromFile(t *testing.T) {
 	defer testutils.VerifyGoLeaksOnce(t)
 	t.Run("primary client", func(t *testing.T) {