Manual instrumentation support - Phase 1

.github/dependabot.yml

@@ -82,6 +82,15 @@ updates:
     schedule:
       interval: weekly
       day: sunday
+  - package-ecosystem: docker
+    directory: /internal/test/e2e/otelglobal
+    labels:
+      - dependencies
+      - docker
+      - Skip Changelog
+    schedule:
+      interval: weekly
+      day: sunday
   - package-ecosystem: gomod
     directory: /
     labels:

.github/workflows/e2e/k8s/sample-job.yml

@@ -26,6 +26,7 @@ spec:
       - name: auto-instrumentation
         image: otel-go-instrumentation
         imagePullPolicy: IfNotPresent
+        command: ["/otel-go-instrumentation", "-global-impl"]
         env:
         - name: OTEL_GO_AUTO_TARGET_EXE
           value: /sample-app/main

.github/workflows/kind.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         k8s-version: ["v1.26.0"]
-        library: ["nethttp", "nethttp_custom", "gin", "databasesql", "grpc"]
+        library: ["nethttp", "nethttp_custom", "gin", "databasesql", "grpc", "otelglobal"]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repo
@@ -68,7 +68,7 @@ jobs:
           kubectl -n default create -f .github/workflows/e2e/k8s/sample-job.yml
       - name: check job status
         run: |
-          kubectl wait --for=condition=Complete --timeout=60s job/sample-job
+          kubectl wait --for=condition=Complete --timeout=60s job/sample-job || kubectl logs -l app=sample -c auto-instrumentation
       - name: copy telemetry trace output
         run: |
           kubectl cp -c filecp default/test-opentelemetry-collector-0:tmp/trace.json ./internal/test/e2e/${{ matrix.library }}/traces-orig.json

CHANGELOG.md

@@ -10,6 +10,9 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http
 
 ### Added
 
+- The CLI flag `global-impl` is added.
+  This flag, when used, enables the instrumentation of the OpenTelemetry default global implementation (https://pkg.go.dev/go.opentelemetry.io/otel).
+  This means that all trace telemetry from this implementation that would normally be dropped will instead be recorded with the auto-instrumentation pipeline. ([#523]https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/523)
 - Add `WithResourceAttributes` `InstrumentationOption` to configure `Instrumentation` to add additional resource attributes. ([#522](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/522))
 
 ### Changed

Makefile

@@ -142,12 +142,13 @@ license-header-check:
 	           exit 1; \
 	   fi
 
-.PHONY: fixture-nethttp fixture-gin fixture-databasesql fixture-nethttp-custom
+.PHONY: fixture-nethttp fixture-gin fixture-databasesql fixture-nethttp-custom fixture-otelglobal
 fixture-nethttp-custom: fixtures/nethttp_custom
 fixture-nethttp: fixtures/nethttp
 fixture-gin: fixtures/gin
 fixture-databasesql: fixtures/databasesql
 fixture-grpc: fixtures/grpc
+fixture-otelglobal: fixtures/otelglobal
 fixtures/%: LIBRARY=$*
 fixtures/%:
 	$(MAKE) docker-build
@@ -161,7 +162,7 @@ fixtures/%:
 	helm install test -f .github/workflows/e2e/k8s/collector-helm-values.yml opentelemetry-helm-charts/charts/opentelemetry-collector
 	kubectl wait --for=condition=Ready --timeout=60s pod/test-opentelemetry-collector-0
 	kubectl -n default create -f .github/workflows/e2e/k8s/sample-job.yml
-	kubectl wait --for=condition=Complete --timeout=60s job/sample-job
+	kubectl wait --for=condition=Complete --timeout=60s job/sample-job || kubectl logs -l app=sample -c auto-instrumentation
 	kubectl cp -c filecp default/test-opentelemetry-collector-0:tmp/trace.json ./internal/test/e2e/$(LIBRARY)/traces-orig.json
 	rm -f ./internal/test/e2e/$(LIBRARY)/traces.json
 	bats ./internal/test/e2e/$(LIBRARY)/verify.bats

cli/main.go

@@ -67,6 +67,9 @@ func newLogger() logr.Logger {
 }
 
 func main() {
+	var globalImpl bool
+
+	flag.BoolVar(&globalImpl, "global-impl", false, "Record telemetry from the OpenTelemetry default global implementation")
 	flag.Usage = usage
 	flag.Parse()
 
@@ -88,8 +91,14 @@ func main() {
 		}
 	}()
 
-	logger.Info("building OpenTelemetry Go instrumentation ...")
-	inst, err := auto.NewInstrumentation(ctx, auto.WithEnv())
+	logger.Info("building OpenTelemetry Go instrumentation ...", "globalImpl", globalImpl)
+
+	instOptions := []auto.InstrumentationOption{auto.WithEnv()}
+	if globalImpl {
+		instOptions = append(instOptions, auto.WithGlobal())
+	}
+
+	inst, err := auto.NewInstrumentation(ctx, instOptions...)
 	if err != nil {
 		logger.Error(err, "failed to create instrumentation")
 		return

examples/go.mod

@@ -7,4 +7,7 @@ require (
 	go.uber.org/zap v1.26.0
 )
 
-require go.uber.org/multierr v1.10.0 // indirect
+require (
+	github.com/stretchr/testify v1.8.4 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+)

examples/go.sum

@@ -2,7 +2,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
 github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=

instrumentation.go

@@ -114,7 +114,7 @@ func NewInstrumentation(ctx context.Context, opts ...InstrumentationOption) (*In
 		return nil, err
 	}
 
-	mngr, err := instrumentation.NewManager(logger, ctrl)
+	mngr, err := instrumentation.NewManager(logger, ctrl, c.globalImpl)
 	if err != nil {
 		return nil, err
 	}
@@ -170,6 +170,7 @@ type instConfig struct {
 	target             process.TargetArgs
 	serviceName        string
 	additionalResAttrs []attribute.KeyValue
+	globalImpl         bool
 }
 
 func newInstConfig(ctx context.Context, opts []InstrumentationOption) (instConfig, error) {
@@ -417,6 +418,30 @@ func WithSampler(sampler trace.Sampler) InstrumentationOption {
 	})
 }
 
+// WithGlobal returns an [InstrumentationOption] that will configure an
+// [Instrumentation] to record telemetry from the [OpenTelemetry default global
+// implementation]. By default, the OpenTelemetry global implementation is a
+// no-op implementation of the OpenTelemetry API. However, by using this
+// option, all telemetry that would have been dropped by the global
+// implementation will be recorded using telemetry pipelines from the
+// configured [Instrumentation].
+//
+// If the target process overrides the default global implementation (e.g.
+// [otel.SetTracerProvider]), the telemetry from that process will go to the
+// set implementation. It will not be recorded using the telemetry pipelines
+// from the configured [Instrumentation] even if this option is used.
+//
+// The OpenTelemetry default global implementation is left unchanged (i.e. it
+// remains a no-op implementation) if this options is not used.
+//
+// [OpenTelemetry default global implementation]: https://pkg.go.dev/go.opentelemetry.io/otel
+func WithGlobal() InstrumentationOption {
+	return fnOpt(func(_ context.Context, c instConfig) (instConfig, error) {
+		c.globalImpl = true
+		return c, nil
+	})
+}
+
 // WithResourceAttributes returns an [InstrumentationOption] that will configure
 // an [Instrumentation] to add the provided attributes to the OpenTelemetry resource.
 func WithResourceAttributes(attrs ...attribute.KeyValue) InstrumentationOption {

internal/include/otel_types.h

@@ -0,0 +1,143 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef _OTEL_TYPES_H
+#define _OTEL_TYPES_H
+
+#include "go_types.h"
+#include "common.h"
+
+// Injected in init
+volatile const u64 attr_type_invalid;
+
+volatile const u64 attr_type_bool;
+volatile const u64 attr_type_int64;
+volatile const u64 attr_type_float64;
+volatile const u64 attr_type_string;
+
+volatile const u64 attr_type_boolslice;
+volatile const u64 attr_type_int64slice;
+volatile const u64 attr_type_float64slice;
+volatile const u64 attr_type_stringslice;
+
+/* Defintions should mimic structs defined in go.opentelemetry.io/otel/attribute */
+
+typedef struct go_otel_attr_value {
+	u64              vtype;
+	u64              numeric;
+	struct go_string string;
+	struct go_iface	 slice;
+} go_otel_attr_value_t;
+
+typedef struct go_otel_key_value {
+	struct go_string     key;
+	go_otel_attr_value_t value;
+} go_otel_key_value_t;
+
+#define OTEL_ATTRIBUTE_KEY_MAX_LEN      (32)
+#define OTEL_ATTRIBUTE_VALUE_MAX_LEN    (128)
+#define OTEL_ATTRUBUTE_MAX_COUNT        (16)
+
+typedef struct otel_attirbute {
+	u16 val_length;
+	u8 vtype;
+	u8 reserved;
+	char key[OTEL_ATTRIBUTE_KEY_MAX_LEN];
+	char value[OTEL_ATTRIBUTE_VALUE_MAX_LEN];
+} otel_attirbute_t;
+
+typedef struct otel_attributes {
+	otel_attirbute_t attrs[OTEL_ATTRUBUTE_MAX_COUNT];
+	u8 valid_attrs;
+}__attribute__((packed)) otel_attributes_t;
+
+static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_value_t *go_attr_value)
+{
+	u64 vtype = go_attr_value->vtype;
+
+	if (vtype == attr_type_invalid) {
+		bpf_printk("Invalid attribute value type\n");
+		return false;
+	}
+
+	// Constant size values
+	if (vtype == attr_type_bool ||
+		vtype == attr_type_int64 ||
+		vtype == attr_type_float64) {
+		bpf_probe_read(attr->value, sizeof(s64), &go_attr_value->numeric);
+		return true;
+	}
+
+	// String values
+	if (vtype == attr_type_string) {
+		if (go_attr_value->string.len >= OTEL_ATTRIBUTE_VALUE_MAX_LEN) {
+			bpf_printk("Aattribute string value is too long\n");
+			return false;
+		}
+		return get_go_string_from_user_ptr(&go_attr_value->string, attr->value, OTEL_ATTRIBUTE_VALUE_MAX_LEN);
+	}
+
+	// TODO (#525): handle slices
+	return false;
+}
+
+static __always_inline void convert_go_otel_attributes(void *attrs_buf, s64 slice_len, otel_attributes_t *enc_attrs)
+{
+	if (attrs_buf == NULL || enc_attrs == NULL){
+		return;
+	}
+
+	if (slice_len < 1) {
+		return;
+	}
+
+	s64 num_attrs = slice_len < OTEL_ATTRUBUTE_MAX_COUNT ? slice_len : OTEL_ATTRUBUTE_MAX_COUNT;
+	go_otel_key_value_t *go_attr = (go_otel_key_value_t*)attrs_buf;
+	go_otel_attr_value_t go_attr_value = {0};
+	struct go_string go_str = {0};
+	u8 valid_attrs = 0;
+
+	for (u32 go_attr_index = 0; go_attr_index < num_attrs; go_attr_index++) {
+		__builtin_memset(&go_attr_value, 0, sizeof(go_otel_attr_value_t));
+		// Read the value struct
+		bpf_probe_read(&go_attr_value, sizeof(go_otel_attr_value_t), &go_attr[go_attr_index].value);
+
+		if (go_attr_value.vtype == attr_type_invalid) {
+			continue;
+		}
+
+		// Read the key string
+		bpf_probe_read(&go_str, sizeof(struct go_string), &go_attr[go_attr_index].key);
+		if (go_str.len >= OTEL_ATTRIBUTE_KEY_MAX_LEN) {
+			// key string is too large
+			bpf_printk("Attribute key string is too long\n");
+			continue;
+		}
+
+		if (!get_go_string_from_user_ptr(&go_str, enc_attrs->attrs[valid_attrs].key, OTEL_ATTRIBUTE_KEY_MAX_LEN)) {
+			continue;
+		}
+
+		if (!set_attr_value(&enc_attrs->attrs[valid_attrs], &go_attr_value)) {
+			continue;
+		}
+
+		enc_attrs->attrs[valid_attrs].vtype = go_attr_value.vtype;
+		valid_attrs++;
+	}
+
+	enc_attrs->valid_attrs = valid_attrs;
+}
+
+#endif