Enable auto/sdk in otel global (#1405)

* Enable auto/sdk for the OTel global probe

Set the auto/sdk flag in the global API to true when newSpan is called.

* Update the otelglobal e2e test

Verify the auto/sdk probe is being used.

* Add a changelog entry

* Unload the newSpan probe after first call

* Update internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/probe.go

* Add package constraints to otelglobal uprobes

* Remove converter.uprobeNewStartMu

* Test SpanFromContext is also supported

The span embedded within a context will be from `auto/sdk`. This means
that #974 is resolved by this change.

Author: MrAlias

CHANGELOG.md

@@ -11,6 +11,7 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http
 ### Added
 - Update `httpPlusdb` demo with adding `OTEL_GO_AUTO_PARSE_DB_STATEMENT` env variable ([#1523](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1523))
 - Support `SELECT`, `INSERT`, `UPDATE`, and `DELETE` for database span names and `db.operation.name` attribute. ([#1253](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1253))
+- Support the full tracing API with the `otelglobal` probe. ([#1405](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1405))
 - Support `go.opentelemetry.io/[email protected]`. ([#1417](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1417))
 - Support `google.golang.org/grpc` `1.69.0`. ([#1417](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1417))
 - Update `google.golang.org/grpc` probe to work with versions `>= 1.69.0`. ([#1431](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1431))

internal/pkg/instrumentation/bpf/database/sql/probe.go

@@ -53,7 +53,7 @@ func New(logger *slog.Logger, version string) probe.Probe {
 					Val: shouldIncludeDBStatement(),
 				},
 			},
-			Uprobes: []probe.Uprobe{
+			Uprobes: []*probe.Uprobe{
 				{
 					Sym:         "database/sql.(*DB).queryDC",
 					EntryProbe:  "uprobe_queryDC",

internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/consumer/probe.go

@@ -69,7 +69,7 @@ func New(logger *slog.Logger, version string) probe.Probe {
 					Val: structfield.NewID("github.com/segmentio/kafka-go", "github.com/segmentio/kafka-go", "ReaderConfig", "GroupID"),
 				},
 			},
-			Uprobes: []probe.Uprobe{
+			Uprobes: []*probe.Uprobe{
 				{
 					Sym:         "github.com/segmentio/kafka-go.(*Reader).FetchMessage",
 					EntryProbe:  "uprobe_FetchMessage",

internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/producer/probe.go

@@ -57,7 +57,7 @@ func New(logger *slog.Logger, version string) probe.Probe {
 					Val: structfield.NewID("github.com/segmentio/kafka-go", "github.com/segmentio/kafka-go", "Message", "Time"),
 				},
 			},
-			Uprobes: []probe.Uprobe{
+			Uprobes: []*probe.Uprobe{
 				{
 					Sym:         "github.com/segmentio/kafka-go.(*Writer).WriteMessages",
 					EntryProbe:  "uprobe_WriteMessages",

internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/probe.go

@@ -60,7 +60,7 @@ func New(logger *slog.Logger) probe.Probe {
 					),
 				},
 			},
-			Uprobes: []probe.Uprobe{
+			Uprobes: []*probe.Uprobe{
 				{
 					Sym:        "go.opentelemetry.io/auto/sdk.(*tracer).start",
 					EntryProbe: "uprobe_Tracer_start",
@@ -86,25 +86,25 @@ type converter struct {
 	logger *slog.Logger
 }
 
-func (c *converter) decodeEvent(record perf.Record) (event, error) {
+func (c *converter) decodeEvent(record perf.Record) (*event, error) {
 	reader := bytes.NewReader(record.RawSample)
 
 	var e event
 	err := binary.Read(reader, binary.LittleEndian, &e.Size)
 	if err != nil {
 		c.logger.Error("failed to decode size", "error", err)
-		return event{}, err
+		return nil, err
 	}
 	c.logger.Debug("decoded size", "size", e.Size)
 
 	e.SpanData = make([]byte, e.Size)
 	_, err = reader.Read(e.SpanData)
 	if err != nil {
 		c.logger.Error("failed to read span data", "error", err)
-		return event{}, err
+		return nil, err
 	}
 	c.logger.Debug("decoded span data", "size", e.Size)
-	return e, nil
+	return &e, nil
 }
 
 func (c *converter) processFn(e *event) ptrace.ScopeSpans {

internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/bpf/probe.bpf.c

@@ -23,6 +23,9 @@ char __license[] SEC("license") = "Dual MIT/GPL";
 #define MAX_BUCKETS 8
 #define MAX_TRACERS 64
 
+// Records state of our write to auto-instrumentation flag.
+bool wrote_flag = false;
+
 struct span_description_t {
     char buf[MAX_STATUS_DESCRIPTION_LEN];
 };
@@ -42,7 +45,12 @@ typedef struct tracer_id {
     char schema_url[MAX_TRACER_SCHEMA_URL_LEN];
 } tracer_id_t;
 
+struct control_t {
+    u32 kind; // Required to be 1.
+};
+
 struct otel_span_t {
+    u32 kind; // Required to be 0.
     BASE_SPAN_PROPERTIES
     struct span_name_t span_name;
     otel_status_t status;
@@ -388,6 +396,36 @@ static __always_inline long fill_tracer_id(tracer_id_t *tracer_id, go_tracer_ptr
     return 0;
 }
 
+// This instrumentation attaches uprobe to the following function:
+// func (t *tracer) newSpan(ctx context.Context, autoSpan *bool, name string, opts []trace.SpanStartOption) (context.Context, trace.Span) {
+// https://github.com/open-telemetry/opentelemetry-go/blob/ac386f383cdfc14f546b4e55e8726a0a45e8a409/internal/global/trace.go#L161
+SEC("uprobe/newSpan")
+int uprobe_newStart(struct pt_regs *ctx) {
+    if (wrote_flag) {
+        // Already wrote flag value.
+        return 0;
+    }
+
+    void *flag_ptr = get_argument(ctx, 4);
+    if (flag_ptr == NULL) {
+        bpf_printk("invalid flag_ptr: NULL");
+        return -1;
+    }
+
+    bool true_value = true;
+    long res = bpf_probe_write_user(flag_ptr, &true_value, sizeof(bool));
+    if (res != 0) {
+        bpf_printk("failed to write bool flag value: %ld", res);
+        return -2;
+    }
+
+    wrote_flag = true;
+
+    // Signal this uprobe should be unloaded.
+    struct control_t ctrl = {1};
+    return bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, (void *)(&ctrl), sizeof(struct control_t));
+}
+
 // This instrumentation attaches uprobe to the following function:
 // func (t *tracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span)
 // https://github.com/open-telemetry/opentelemetry-go/blob/98b32a6c3a87fbee5d34c063b9096f416b250897/internal/global/trace.go#L149

internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/bpf_arm64_bpfel.go

@@ -4,6 +4,7 @@
 package global
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"log/slog"
@@ -18,6 +19,7 @@ import (
 	"go.opentelemetry.io/auto/internal/pkg/process"
 	"go.opentelemetry.io/auto/internal/pkg/structfield"
 
+	"github.com/cilium/ebpf/perf"
 	"github.com/hashicorp/go-version"
 	"golang.org/x/sys/unix"
 
@@ -33,6 +35,27 @@ import (
 const (
 	// pkg is the package being instrumented.
 	pkg = "go.opentelemetry.io/otel/internal/global"
+
+	// Minimum version of go.opentelemetry.io/otel that supports using the
+	// go.opentelemetry.io/auto/sdk in the global API.
+	minAutoSDK = "1.33.0"
+)
+
+var (
+	otelWithAutoSDK = probe.PackageConstrainst{
+		Package: "go.opentelemetry.io/otel",
+		Constraints: version.MustConstraints(
+			version.NewConstraint(fmt.Sprintf(">= %s", minAutoSDK)),
+		),
+		FailureMode: probe.FailureModeIgnore,
+	}
+	otelWithoutAutoSDK = probe.PackageConstrainst{
+		Package: "go.opentelemetry.io/otel",
+		Constraints: version.MustConstraints(
+			version.NewConstraint(fmt.Sprintf("< %s", minAutoSDK)),
+		),
+		FailureMode: probe.FailureModeIgnore,
+	}
 )
 
 // New returns a new [probe.Probe].
@@ -41,6 +64,19 @@ func New(logger *slog.Logger) probe.Probe {
 		SpanKind:        trace.SpanKindClient,
 		InstrumentedPkg: pkg,
 	}
+
+	uprobeNewStart := &probe.Uprobe{
+		Sym:        "go.opentelemetry.io/otel/internal/global.(*tracer).newSpan",
+		EntryProbe: "uprobe_newStart",
+		PackageConstrainsts: []probe.PackageConstrainst{
+			otelWithAutoSDK,
+		},
+	}
+
+	c := &converter{
+		logger:         logger,
+		uprobeNewStart: uprobeNewStart,
+	}
 	return &probe.TraceProducer[bpfObjects, event]{
 		Base: probe.Base[bpfObjects, event]{
 			ID:     id,
@@ -107,38 +143,94 @@ func New(logger *slog.Logger) probe.Probe {
 				tracerIDContainsSchemaURL{},
 				tracerIDContainsScopeAttributes{},
 			},
-			Uprobes: []probe.Uprobe{
+			Uprobes: []*probe.Uprobe{
+				uprobeNewStart,
 				{
 					Sym:         "go.opentelemetry.io/otel/internal/global.(*tracer).Start",
 					EntryProbe:  "uprobe_Start",
 					ReturnProbe: "uprobe_Start_Returns",
+					PackageConstrainsts: []probe.PackageConstrainst{
+						otelWithoutAutoSDK,
+					},
 				},
 				{
 					Sym:        "go.opentelemetry.io/otel/internal/global.(*nonRecordingSpan).End",
 					EntryProbe: "uprobe_End",
+					PackageConstrainsts: []probe.PackageConstrainst{
+						otelWithoutAutoSDK,
+					},
 				},
 				{
 					Sym:         "go.opentelemetry.io/otel/internal/global.(*nonRecordingSpan).SetAttributes",
 					EntryProbe:  "uprobe_SetAttributes",
 					FailureMode: probe.FailureModeIgnore,
+					PackageConstrainsts: []probe.PackageConstrainst{
+						otelWithoutAutoSDK,
+					},
 				},
 				{
 					Sym:         "go.opentelemetry.io/otel/internal/global.(*nonRecordingSpan).SetStatus",
 					EntryProbe:  "uprobe_SetStatus",
 					FailureMode: probe.FailureModeIgnore,
+					PackageConstrainsts: []probe.PackageConstrainst{
+						otelWithoutAutoSDK,
+					},
 				},
 				{
 					Sym:         "go.opentelemetry.io/otel/internal/global.(*nonRecordingSpan).SetName",
 					EntryProbe:  "uprobe_SetName",
 					FailureMode: probe.FailureModeIgnore,
+					PackageConstrainsts: []probe.PackageConstrainst{
+						otelWithoutAutoSDK,
+					},
 				},
 			},
-			SpecFn: loadBpf,
+			SpecFn:        loadBpf,
+			ProcessRecord: c.decodeEvent,
 		},
 		ProcessFn: processFn,
 	}
 }
 
+type recordKind uint32
+
+const (
+	recordKindTelemetry recordKind = iota
+	recordKindConrol
+)
+
+type converter struct {
+	logger *slog.Logger
+
+	uprobeNewStart *probe.Uprobe
+}
+
+func (c *converter) decodeEvent(record perf.Record) (*event, error) {
+	reader := bytes.NewReader(record.RawSample)
+
+	var kind recordKind
+	err := binary.Read(reader, binary.LittleEndian, &kind)
+	if err != nil {
+		return nil, err
+	}
+
+	var e *event
+	switch kind {
+	case recordKindTelemetry:
+		e = new(event)
+		reader.Reset(record.RawSample)
+		err = binary.Read(reader, binary.LittleEndian, e)
+	case recordKindConrol:
+		if c.uprobeNewStart != nil {
+			err = c.uprobeNewStart.Close()
+			c.uprobeNewStart = nil
+		}
+	default:
+		err = fmt.Errorf("unknown record kind: %d", kind)
+	}
+	return e, err
+}
+
 // tracerIDContainsSchemaURL is a Probe Const defining whether the tracer key contains schemaURL.
 type tracerIDContainsSchemaURL struct{}