Failing to push OpenTelemtry metrics and tracing to OTLP endpoint

I’m trying to connect to NR directly but it’s constantly logging:

2021/11/01 19:00:14 context deadline exceeded                                                                                                                                                                                         
2021/11/01 19:00:14 metrics exporter is disconnected from the server otlp.nr-data.net:4317: context deadline exceeded                                                                                                                  │
2021/11/01 19:00:14 metrics exporter is disconnected from the server otlp.nr-data.net:4317: context deadline exceeded

Connecting to OTLP collector seems to work fine. My Code snippet is below:

var res, err = getResourceAttributes(serviceName)
if err != nil {
	logger.Fatal("Failed to initialized trace source", zap.Error(err))
}

var apiKey = os.Getenv(ApiKey)
var clientOpts = []otlptracegrpc.Option{
	otlptracegrpc.WithEndpoint(endpoint),
	otlptracegrpc.WithInsecure(),
	otlptracegrpc.WithReconnectionPeriod(2 * time.Second),
	otlptracegrpc.WithDialOption(grpc.WithBlock()),
	otlptracegrpc.WithTimeout(30 * time.Second),
}
if strings.TrimSpace(apiKey) != "" {
	var headers = map[string]string{
		"api-key": apiKey,
	}
	clientOpts = append(
		clientOpts, otlptracegrpc.WithHeaders(headers),
		otlptracegrpc.WithCompressor("gzip"),
	)
	logger.Info("Metrics provider Using API Key", zap.String("key", apiKey))
}
var client = otlptracegrpc.NewClient(clientOpts...)
var traceExporter sdkTrace.SpanExporter
traceExporter, err = otlptrace.New(context.Background(), client)
if err != nil {
	logger.Fatal("Failed to initialized trace exporter", zap.Error(err))
}
logger.Info("Connected to OTLP", zap.String("endpoint", endpoint))
var tracerProvider = sdkTrace.NewTracerProvider(
	sdkTrace.WithSampler(sdkTrace.AlwaysSample()),
	sdkTrace.WithResource(res),
	sdkTrace.WithBatcher(traceExporter),
)

// Set Global Trace Provider
otel.SetTracerProvider(tracerProvider)

// set global propagator to trace context (the default is no-op).
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
1 Like

I tried using either

https://otlp.nr-data.net:4317

or

otlp.nr-data.net:4317

Neither worked.

1 Like

A quick example to reproduce it

package main

import (
	"context"
	"fmt"
	"github.com/google/uuid"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/sdk/resource"
	sdkTrace "go.opentelemetry.io/otel/sdk/trace"
	semConv "go.opentelemetry.io/otel/semconv/v1.7.0"
	"google.golang.org/grpc"
	"google.golang.org/grpc/encoding/gzip"
	"log"
	"time"

	"go.opentelemetry.io/otel"
)

const (
	serviceName  = "testing-service"
	otlpEndpoint = "otlp.nr-data.net:4317"
	apiKey = ""
)

func main() {

	var res, _ = resource.New(context.Background(),
		resource.WithAttributes(
			semConv.ServiceNameKey.String(serviceName),
			semConv.ServiceInstanceIDKey.String(uuid.New().String()),
			semConv.TelemetrySDKLanguageGo,
		),
	)
	var header = map[string]string{
		"api-key": apiKey,
	}
	ctx := context.Background()
	client := otlptracegrpc.NewClient(
		otlptracegrpc.WithInsecure(),
		otlptracegrpc.WithEndpoint(otlpEndpoint),
		otlptracegrpc.WithHeaders(header),
		otlptracegrpc.WithCompressor(gzip.Name),
		otlptracegrpc.WithDialOption(grpc.WithBlock()),
		otlptracegrpc.WithReconnectionPeriod(time.Second),
	)
	exp, err := otlptrace.New(ctx, client)
	if err != nil {
		log.Fatalf("Failed to create the collector exporter: %v", err)
	}
	defer func() {
		ctx, cancel := context.WithTimeout(ctx, time.Second)
		defer cancel()
		if err := exp.Shutdown(ctx); err != nil {
			otel.Handle(err)
		}
	}()

	tracerProvider := sdkTrace.NewTracerProvider(
		sdkTrace.WithBatcher(exp,
			sdkTrace.WithBatchTimeout(5*time.Second),
			sdkTrace.WithExportTimeout(5*time.Second),
			sdkTrace.WithMaxQueueSize(10000),
			sdkTrace.WithMaxExportBatchSize(100000),
		),
		sdkTrace.WithResource(res),
	)
	otel.SetTracerProvider(tracerProvider)

	defer func() {
		if err := tracerProvider.Shutdown(ctx); err != nil {
			log.Fatalf("stopping tracer provider: %v", err)
		}
	}()

	var tracer = otel.GetTracerProvider().Tracer("testing")

	for i := 0; i < 10; i++ {
		var _, span = tracer.Start(context.Background(), fmt.Sprintf("trace %v", i))
		time.Sleep(time.Second)
		span.End()
	}

	time.Sleep(10 * time.Second)
}

Hi. I am having the same problem. Did you find a solution?

@manuel.ledesma I have solved this problem. It is necessary to create a grpc connection with the TLS.

	ctx := context.Background()
	pool, err := x509.SystemCertPool()
	if err != nil {
		panic(err)
	}
	creds := credentials.NewClientTLSFromCert(pool, "")
	texp, err := otlptracegrpc.New(
		ctx,
		otlptracegrpc.WithHeaders(map[string]string{"api-key": "*********"}),
		otlptracegrpc.WithEndpoint("otlp.nr-data.net:4317"),
		otlptracegrpc.WithTLSCredentials(creds),
	)
	if err != nil {
		panic(err)
	}
1 Like

We were using as workaround the OTEL collector, which is also having the same issue now, which I’m assuming it’s probably using the same approach internally, I will give a try to this approach to see if it makes a difference.

Thanks.