feat: support multiple trace agents (#1183)

* feat: support multiple trace agents

* feat: support multiple trace agents, let later calls run if error happens

* test: add more tests

core/trace/agent.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/tal-tech/go-zero/core/lang"
 	"github.com/tal-tech/go-zero/core/logx"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/jaeger"
@@ -19,17 +20,31 @@ const (
 	kindZipkin = "zipkin"
 )
 
-var once sync.Once
+var (
+	agents = make(map[string]lang.PlaceholderType)
+	lock   sync.Mutex
+)
 
 // StartAgent starts a opentelemetry agent.
 func StartAgent(c Config) {
-	once.Do(func() {
+	lock.Lock()
-		startAgent(c)
+	defer lock.Unlock()
-	})
+
+	_, ok := agents[c.Endpoint]
+	if ok {
+		return
+	}
+
+	// if error happens, let later calls run.
+	if err := startAgent(c); err != nil {
+		return
+	}
+
+	agents[c.Endpoint] = lang.Placeholder
 }
 
 func createExporter(c Config) (sdktrace.SpanExporter, error) {
-	// Just support jaeger now, more for later
+	// Just support jaeger and zipkin now, more for later
 	switch c.Batcher {
 	case kindJaeger:
 		return jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(c.Endpoint)))
@@ -40,7 +55,7 @@ func createExporter(c Config) (sdktrace.SpanExporter, error) {
 	}
 }
 
-func startAgent(c Config) {
+func startAgent(c Config) error {
 	opts := []sdktrace.TracerProviderOption{
 		// Set the sampling rate based on the parent span to 100%
 		sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(c.Sampler))),
@@ -52,7 +67,7 @@ func startAgent(c Config) {
 		exp, err := createExporter(c)
 		if err != nil {
 			logx.Error(err)
-			return
+			return err
 		}
 
 		// Always be sure to batch in production.
@@ -66,4 +81,6 @@ func startAgent(c Config) {
 	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
 		logx.Errorf("[otel] error: %v", err)
 	}))
+
+	return nil
 }

core/trace/agent_test.go

@@ -0,0 +1,54 @@
+package trace
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/logx"
+)
+
+func TestStartAgent(t *testing.T) {
+	logx.Disable()
+
+	const (
+		endpoint1 = "localhost:1234"
+		endpoint2 = "remotehost:1234"
+		endpoint3 = "localhost:1235"
+	)
+	c1 := Config{
+		Name: "foo",
+	}
+	c2 := Config{
+		Name:     "bar",
+		Endpoint: endpoint1,
+		Batcher:  kindJaeger,
+	}
+	c3 := Config{
+		Name:     "any",
+		Endpoint: endpoint2,
+		Batcher:  kindZipkin,
+	}
+	c4 := Config{
+		Name:     "bla",
+		Endpoint: endpoint3,
+		Batcher:  "otlp",
+	}
+
+	StartAgent(c1)
+	StartAgent(c1)
+	StartAgent(c2)
+	StartAgent(c3)
+	StartAgent(c4)
+
+	lock.Lock()
+	defer lock.Unlock()
+
+	// because remotehost cannot be resolved
+	assert.Equal(t, 2, len(agents))
+	_, ok := agents[""]
+	assert.True(t, ok)
+	_, ok = agents[endpoint1]
+	assert.True(t, ok)
+	_, ok = agents[endpoint2]
+	assert.False(t, ok)
+}