Include resource attributes as target_info for Prometheus exporters (#5407)

Author: robertcoltheart

src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+* Fix serializing scope_info when buffer overflows
+  ([#5407](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5407))
+
+* Add `target_info` to Prometheus exporters when using OpenMetrics
+  ([#5407](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5407))
+
 ## 1.8.0-beta.1
 
 Released 2024-Mar-14
@@ -10,6 +16,7 @@ Released 2024-Mar-14
   ([#5305](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5305))
 
 * Export OpenMetrics format from Prometheus exporters ([#5107](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5107))
+
 * For requests with OpenMetrics format, scope info is automatically added
   ([#5086](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5086)
   [#5182](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5182))

src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+* Fix serializing scope_info when buffer overflows
+  ([#5407](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5407))
+
+* Add `target_info` to Prometheus exporters when using OpenMetrics
+  ([#5407](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5407))
+
 ## 1.8.0-beta.1
 
 Released 2024-Mar-14

src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusCollectionManager.cs

@@ -17,6 +17,7 @@ internal sealed class PrometheusCollectionManager
     private readonly HashSet<string> scopes;
     private int metricsCacheCount;
     private byte[] buffer = new byte[85000]; // encourage the object to live in LOH (large object heap)
+    private int targetInfoBufferLength = -1; // zero or positive when target_info has been written for the first time
     private int globalLockState;
     private ArraySegment<byte> previousDataView;
     private DateTime? previousDataViewGeneratedAtUtc;
@@ -174,13 +175,20 @@ private ExportResult OnCollect(Batch<Metric> metrics)
         {
             if (this.exporter.OpenMetricsRequested)
             {
+                cursor = this.WriteTargetInfo();
+
                 this.scopes.Clear();
 
                 foreach (var metric in metrics)
                 {
-                    if (PrometheusSerializer.CanWriteMetric(metric))
+                    if (!PrometheusSerializer.CanWriteMetric(metric))
+                    {
+                        continue;
+                    }
+
+                    if (this.scopes.Add(metric.MeterName))
                     {
-                        if (this.scopes.Add(metric.MeterName))
+                        while (true)
                         {
                             try
                             {
@@ -262,6 +270,31 @@ private ExportResult OnCollect(Batch<Metric> metrics)
         }
     }
 
+    private int WriteTargetInfo()
+    {
+        if (this.targetInfoBufferLength < 0)
+        {
+            while (true)
+            {
+                try
+                {
+                    this.targetInfoBufferLength = PrometheusSerializer.WriteTargetInfo(this.buffer, 0, this.exporter.Resource);
+
+                    break;
+                }
+                catch (IndexOutOfRangeException)
+                {
+                    if (!this.IncreaseBufferSize())
+                    {
+                        throw;
+                    }
+                }
+            }
+        }
+
+        return this.targetInfoBufferLength;
+    }
+
     private bool IncreaseBufferSize()
     {
         var newBufferSize = this.buffer.Length * 2;

src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusExporter.cs

@@ -3,6 +3,7 @@
 
 using OpenTelemetry.Internal;
 using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
 
 namespace OpenTelemetry.Exporter.Prometheus;
 
@@ -14,6 +15,7 @@ internal sealed class PrometheusExporter : BaseExporter<Metric>, IPullMetricExpo
 {
     private Func<int, bool> funcCollect;
     private Func<Batch<Metric>, ExportResult> funcExport;
+    private Resource resource;
     private bool disposed;
 
     /// <summary>
@@ -55,6 +57,8 @@ internal Func<Batch<Metric>, ExportResult> OnExport
 
     internal bool OpenMetricsRequested { get; set; }
 
+    internal Resource Resource => this.resource ??= this.ParentProvider.GetResource();
+
     /// <inheritdoc/>
     public override ExportResult Export(in Batch<Metric> metrics)
     {

src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializer.cs

@@ -6,6 +6,7 @@
 using System.Runtime.CompilerServices;
 using OpenTelemetry.Internal;
 using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
 
 namespace OpenTelemetry.Exporter.Prometheus;
 
@@ -396,6 +397,40 @@ public static int WriteTags(byte[] buffer, int cursor, Metric metric, ReadOnlyTa
         return cursor;
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static int WriteTargetInfo(byte[] buffer, int cursor, Resource resource)
+    {
+        if (resource == Resource.Empty)
+        {
+            return cursor;
+        }
+
+        cursor = WriteAsciiStringNoEscape(buffer, cursor, "# TYPE target info");
+        buffer[cursor++] = ASCII_LINEFEED;
+
+        cursor = WriteAsciiStringNoEscape(buffer, cursor, "# HELP target Target metadata");
+        buffer[cursor++] = ASCII_LINEFEED;
+
+        cursor = WriteAsciiStringNoEscape(buffer, cursor, "target_info");
+        buffer[cursor++] = unchecked((byte)'{');
+
+        foreach (var attribute in resource.Attributes)
+        {
+            cursor = WriteLabel(buffer, cursor, attribute.Key, attribute.Value);
+
+            buffer[cursor++] = unchecked((byte)',');
+        }
+
+        cursor--; // Write over the last written comma
+
+        buffer[cursor++] = unchecked((byte)'}');
+        buffer[cursor++] = unchecked((byte)' ');
+        buffer[cursor++] = unchecked((byte)'1');
+        buffer[cursor++] = ASCII_LINEFEED;
+
+        return cursor;
+    }
+
     private static string MapPrometheusType(PrometheusType type)
     {
         return type switch

test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs

@@ -12,6 +12,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
 using OpenTelemetry.Tests;
 using Xunit;
 
@@ -150,6 +151,7 @@ public async Task PrometheusExporterMiddlewareIntegration_MeterProvider()
     {
         using MeterProvider meterProvider = Sdk.CreateMeterProviderBuilder()
             .AddMeter(MeterName)
+            .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1"))
             .AddPrometheusExporter()
             .Build();
 
@@ -213,6 +215,7 @@ public async Task PrometheusExporterMiddlewareIntegration_MapEndpoint_WithMeterP
     {
         using MeterProvider meterProvider = Sdk.CreateMeterProviderBuilder()
             .AddMeter(MeterName)
+            .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1"))
             .AddPrometheusExporter()
             .Build();
 
@@ -265,11 +268,12 @@ private static async Task RunPrometheusExporterMiddlewareIntegrationTest(
                    if (registerMeterProvider)
                    {
                        services.AddOpenTelemetry().WithMetrics(builder => builder
-                            .AddMeter(MeterName)
-                            .AddPrometheusExporter(o =>
-                            {
-                                configureOptions?.Invoke(o);
-                            }));
+                           .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1"))
+                           .AddMeter(MeterName)
+                           .AddPrometheusExporter(o =>
+                           {
+                               configureOptions?.Invoke(o);
+                           }));
                    }
 
                    configureServices?.Invoke(services);
@@ -322,7 +326,10 @@ private static async Task RunPrometheusExporterMiddlewareIntegrationTest(
             string content = await response.Content.ReadAsStringAsync();
 
             string expected = requestOpenMetrics
-                ? "# TYPE otel_scope_info info\n"
+                ? "# TYPE target info\n"
+                  + "# HELP target Target metadata\n"
+                  + "target_info{service_name='my_service',service_instance_id='id1'} 1\n"
+                  + "# TYPE otel_scope_info info\n"
                   + "# HELP otel_scope_info Scope metadata\n"
                   + $"otel_scope_info{{otel_scope_name='{MeterName}'}} 1\n"
                   + "# TYPE counter_double_total counter\n"

test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs

@@ -7,6 +7,7 @@
 using System.Net.Http;
 #endif
 using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
 using OpenTelemetry.Tests;
 using Xunit;
 
@@ -175,6 +176,7 @@ private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetri
             {
                 provider = Sdk.CreateMeterProviderBuilder()
                     .AddMeter(meter.Name)
+                    .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1"))
                     .AddPrometheusHttpListener(options =>
                     {
                         options.UriPrefixes = new string[] { address };
@@ -233,7 +235,10 @@ private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetri
             var content = await response.Content.ReadAsStringAsync();
 
             var expected = requestOpenMetrics
-                ? "# TYPE otel_scope_info info\n"
+                ? "# TYPE target info\n"
+                  + "# HELP target Target metadata\n"
+                  + "target_info{service_name='my_service',service_instance_id='id1'} 1\n"
+                  + "# TYPE otel_scope_info info\n"
                   + "# HELP otel_scope_info Scope metadata\n"
                   + $"otel_scope_info{{otel_scope_name='{MeterName}'}} 1\n"
                   + "# TYPE counter_double_total counter\n"