WIP: Add builtin OpenTelemetry support

cmd/web.go

@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/gtprof"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/otelexporter"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/public"
 	"code.gitea.io/gitea/modules/setting"
@@ -220,7 +221,12 @@ func serveInstalled(ctx *cli.Context) error {
 		}
 	}
 
-	gtprof.EnableBuiltinTracer(util.Iif(setting.IsProd, 2000*time.Millisecond, 100*time.Millisecond))
+	gtprof.EnableTracer(&gtprof.TracerOptions{
+		ServiceName:      "gitea",
+		AppVer:           setting.AppVer,
+		BuiltinThreshold: util.Iif(setting.IsProd, 2000*time.Millisecond, 100*time.Millisecond),
+	})
+	otelexporter.InitDefaultOtelExporter()
 
 	// Set up Chi routes
 	webRoutes := routers.NormalRoutes()

custom/conf/app.example.ini

@@ -2815,3 +2815,16 @@ LEVEL = Info
 ;SERVICE_TYPE = memory
 ;; Ignored for the "memory" type. For "redis" use something like `redis://127.0.0.1:6379/0`
 ;SERVICE_CONN_STR =
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; OpenTelemetry exporter
+;; These are the supported options picked from https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp
+;; This feature is experimental and submit to change
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;[otel_exporter]
+;OTLP_ENABLED = false
+;OTLP_ENDPOINT = http://localhost:4318
+;OTLP_HEADERS =
+;OTLP_TIMEOUT = 10s
+;OTLP_COMPRESSION = gzip
+;OTLP_TLS_INSECURE = false

modules/gtprof/gtprof.go

@@ -3,6 +3,11 @@
 
 package gtprof
 
+import (
+	"sync/atomic"
+	"time"
+)
+
 // This is a Gitea-specific profiling package,
 // the name is chosen to distinguish it from the standard pprof tool and "GNU gprof"
 
@@ -23,3 +28,14 @@ const LabelProcessType = "process_type"
 
 // LabelProcessDescription is a label set on goroutines that have a process attached
 const LabelProcessDescription = "process_description"
+
+type TracerOptions struct {
+	ServiceName, AppVer string
+	BuiltinThreshold    time.Duration
+}
+
+var tracerOptions atomic.Pointer[TracerOptions]
+
+func EnableTracer(opts *TracerOptions) {
+	tracerOptions.Store(opts)
+}

modules/gtprof/trace.go

@@ -5,7 +5,10 @@ package gtprof
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
 	"fmt"
+	mathRand "math/rand/v2"
 	"sync"
 	"time"
 
@@ -36,6 +39,7 @@ type TraceSpan struct {
 
 	// mutable, must be protected by mutex
 	mu         sync.RWMutex
+	id         string
 	name       string
 	statusCode uint32
 	statusDesc string
@@ -54,6 +58,11 @@ type TraceValue struct {
 	v any
 }
 
+func (t *TraceValue) IsString() bool {
+	_, ok := t.v.(string)
+	return ok
+}
+
 func (t *TraceValue) AsString() string {
 	return fmt.Sprint(t.v)
 }
@@ -71,6 +80,7 @@ func (t *TraceValue) AsFloat64() float64 {
 var globalTraceStarters []traceStarter
 
 type Tracer struct {
+	chacha8  *mathRand.ChaCha8
 	starters []traceStarter
 }
 
@@ -113,7 +123,7 @@ func (t *Tracer) Start(ctx context.Context, spanName string) (context.Context, *
 	if starters == nil {
 		starters = globalTraceStarters
 	}
-	ts := &TraceSpan{name: spanName, startTime: time.Now()}
+	ts := &TraceSpan{id: t.randomHexForBytes(8), name: spanName, startTime: time.Now()}
 	parentSpan := GetContextSpan(ctx)
 	if parentSpan != nil {
 		parentSpan.mu.Lock()
@@ -165,8 +175,19 @@ func (s *TraceSpan) End() {
 	}
 }
 
+func (t *Tracer) randomHexForBytes(n int) string {
+	b := make([]byte, n)
+	_, _ = t.chacha8.Read(b) // it never fails
+	return hex.EncodeToString(b)
+}
+
 func GetTracer() *Tracer {
-	return &Tracer{}
+	var seed [32]byte
+	_, err := rand.Read(seed[:])
+	if err != nil {
+		panic(fmt.Sprintf("rand.Read: %v", err))
+	}
+	return &Tracer{chacha8: mathRand.NewChaCha8(seed)}
 }
 
 func GetContextSpan(ctx context.Context) *TraceSpan {

modules/gtprof/trace_builtin.go

@@ -7,8 +7,6 @@ import (
 	"context"
 	"fmt"
 	"strings"
-	"sync/atomic"
-	"time"
 
 	"code.gitea.io/gitea/modules/tailmsg"
 )
@@ -72,12 +70,16 @@ func (t *traceBuiltinSpan) end() {
 	if t.ts.parent == nil {
 		// TODO: debug purpose only
 		// TODO: it should distinguish between http response network lag and actual processing time
-		threshold := time.Duration(traceBuiltinThreshold.Load())
-		if threshold != 0 && t.ts.endTime.Sub(t.ts.startTime) > threshold {
+		opts := tracerOptions.Load()
+		if opts == nil {
+			return
+		}
+		if t.ts.endTime.Sub(t.ts.startTime) > opts.BuiltinThreshold {
 			sb := &strings.Builder{}
 			t.toString(sb, 0)
 			tailmsg.GetManager().GetTraceRecorder().Record(sb.String())
 		}
+		otelRecordTrace(t)
 	}
 }
 
@@ -88,9 +90,3 @@ func (t *traceBuiltinStarter) start(ctx context.Context, traceSpan *TraceSpan, i
 func init() {
 	globalTraceStarters = append(globalTraceStarters, &traceBuiltinStarter{})
 }
-
-var traceBuiltinThreshold atomic.Int64
-
-func EnableBuiltinTracer(threshold time.Duration) {
-	traceBuiltinThreshold.Store(int64(threshold))
-}

modules/gtprof/trace_otel.go

@@ -0,0 +1,71 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package gtprof
+
+import (
+	"strconv"
+
+	"code.gitea.io/gitea/modules/otelexporter"
+)
+
+func otelToSpan(traceID string, t *traceBuiltinSpan, scopeSpan *otelexporter.OtelScopeSpan) {
+	t.ts.mu.RLock()
+	defer t.ts.mu.RUnlock()
+
+	span := &otelexporter.OtelSpan{
+		TraceID:           traceID,
+		SpanID:            t.ts.id,
+		Name:              t.ts.name,
+		StartTimeUnixNano: strconv.FormatInt(t.ts.startTime.UnixNano(), 10),
+		EndTimeUnixNano:   strconv.FormatInt(t.ts.endTime.UnixNano(), 10),
+		Kind:              2,
+	}
+
+	if t.ts.parent != nil {
+		span.ParentSpanID = t.ts.parent.id
+	}
+
+	scopeSpan.Spans = append(scopeSpan.Spans, span)
+
+	for _, a := range t.ts.attributes {
+		var otelVal any
+		if a.Value.IsString() {
+			otelVal = otelexporter.OtelAttributeStringValue{StringValue: a.Value.AsString()}
+		} else {
+			otelVal = otelexporter.OtelAttributeIntValue{IntValue: strconv.FormatInt(a.Value.AsInt64(), 10)}
+		}
+		span.Attributes = append(span.Attributes, &otelexporter.OtelAttribute{Key: a.Key, Value: otelVal})
+	}
+
+	for _, c := range t.ts.children {
+		child := c.internalSpans[t.internalSpanIdx].(*traceBuiltinSpan)
+		otelToSpan(traceID, child, scopeSpan)
+	}
+}
+
+func otelRecordTrace(t *traceBuiltinSpan) {
+	exporter := otelexporter.GetDefaultOtelExporter()
+	if exporter == nil {
+		return
+	}
+	opts := tracerOptions.Load()
+	scopeSpan := &otelexporter.OtelScopeSpan{
+		Scope: &otelexporter.OtelScope{Name: "gitea-server", Version: opts.AppVer},
+	}
+
+	traceID := GetTracer().randomHexForBytes(16)
+	otelToSpan(traceID, t, scopeSpan)
+
+	resSpans := otelexporter.OtelResourceSpan{
+		Resource: &otelexporter.OtelResource{
+			Attributes: []*otelexporter.OtelAttribute{
+				{Key: "service.name", Value: otelexporter.OtelAttributeStringValue{StringValue: opts.ServiceName}},
+			},
+		},
+		ScopeSpans: []*otelexporter.OtelScopeSpan{scopeSpan},
+	}
+
+	otelTrace := &otelexporter.OtelTrace{ResourceSpans: []*otelexporter.OtelResourceSpan{&resSpans}}
+	exporter.ExportTrace(otelTrace)
+}

modules/otelexporter/otelexporter.go

@@ -0,0 +1,81 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package otelexporter
+
+import (
+	"bytes"
+	"net/http"
+	"sync/atomic"
+
+	"code.gitea.io/gitea/modules/json"
+)
+
+type OtelAttributeStringValue struct {
+	StringValue string `json:"stringValue"`
+}
+
+type OtelAttributeIntValue struct {
+	IntValue string `json:"intValue"`
+}
+
+type OtelAttribute struct {
+	Key   string `json:"key"`
+	Value any    `json:"value"`
+}
+
+type OtelResource struct {
+	Attributes []*OtelAttribute `json:"attributes,omitempty"`
+}
+
+type OtelScope struct {
+	Name       string           `json:"name"`
+	Version    string           `json:"version"`
+	Attributes []*OtelAttribute `json:"attributes,omitempty"`
+}
+
+type OtelSpan struct {
+	TraceID           string           `json:"traceId"`
+	SpanID            string           `json:"spanId"`
+	ParentSpanID      string           `json:"parentSpanId,omitempty"`
+	Name              string           `json:"name"`
+	StartTimeUnixNano string           `json:"startTimeUnixNano"`
+	EndTimeUnixNano   string           `json:"endTimeUnixNano"`
+	Kind              int              `json:"kind"`
+	Attributes        []*OtelAttribute `json:"attributes,omitempty"`
+}
+
+type OtelScopeSpan struct {
+	Scope *OtelScope  `json:"scope"`
+	Spans []*OtelSpan `json:"spans"`
+}
+
+type OtelResourceSpan struct {
+	Resource   *OtelResource    `json:"resource"`
+	ScopeSpans []*OtelScopeSpan `json:"scopeSpans"`
+}
+
+type OtelTrace struct {
+	ResourceSpans []*OtelResourceSpan `json:"resourceSpans"`
+}
+
+type OtelExporter struct{}
+
+func (e *OtelExporter) ExportTrace(trace *OtelTrace) {
+	// TODO: use a async queue
+	otelTraceJSON, err := json.Marshal(trace)
+	if err == nil {
+		_, _ = http.Post("http://localhost:4318/v1/traces", "application/json", bytes.NewReader(otelTraceJSON))
+	}
+}
+
+var defaultOtelExporter atomic.Pointer[OtelExporter]
+
+func GetDefaultOtelExporter() *OtelExporter {
+	return defaultOtelExporter.Load()
+}
+
+func InitDefaultOtelExporter() {
+	e := &OtelExporter{}
+	defaultOtelExporter.Store(e)
+}

modules/setting/otel.go

@@ -0,0 +1,57 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package setting
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+)
+
+type OtelExporterStruct struct {
+	OtlpEnabled     bool
+	OtlpEndpoint    string
+	OtlpCompression string
+	OtlpTLSInsecure bool
+
+	OtlpHeaders map[string]string `ini:"-"`
+	OtlpTimeout time.Duration     `ini:"-"`
+}
+
+var OtelExporter OtelExporterStruct
+
+func loadOtelExporterFrom(cfg ConfigProvider) error {
+	OtelExporter = OtelExporterStruct{
+		OtlpEndpoint:    "http://localhost:4318",
+		OtlpCompression: "gzip",
+		OtlpTimeout:     time.Second * 10,
+	}
+	sec := cfg.Section("otel_exporter")
+	if err := sec.MapTo(&OtelExporter); err != nil {
+		return err
+	}
+	if !OtelExporter.OtlpEnabled {
+		return nil
+	}
+
+	OtelExporter.OtlpTimeout = sec.Key("OTLP_TIMEOUT").MustDuration(OtelExporter.OtlpTimeout)
+
+	otlpHeadersString := sec.Key("OTLP_HEADERS").String()
+	if otlpHeadersString != "" {
+		OtelExporter.OtlpHeaders = make(map[string]string)
+		for _, header := range strings.Split(otlpHeadersString, ",") {
+			header = strings.TrimSpace(header)
+			if key, valRaw, ok := strings.Cut(header, "="); ok {
+				val, err := url.QueryUnescape(valRaw)
+				if err != nil {
+					return fmt.Errorf("invalid OTLP_HEADER %q, err: %w", header, err)
+				}
+				OtelExporter.OtlpHeaders[key] = val
+			}
+		}
+	}
+
+	return nil
+}

modules/setting/setting.go

@@ -217,6 +217,9 @@ func LoadSettings() {
 	loadProjectFrom(CfgProvider)
 	loadMimeTypeMapFrom(CfgProvider)
 	loadFederationFrom(CfgProvider)
+	if err := loadOtelExporterFrom(CfgProvider); err != nil {
+		log.Fatal("Unable to load otel_exporter settings: %v", err)
+	}
 }
 
 // LoadSettingsForInstall initializes the settings for install