Skip to content

Commit eb2dc44

Browse files
vishweshbankwarvishweshbankwar
authored
[otlp] Add experimental feature flag for enabling retries (#5435)
1 parent 4cccb09 commit eb2dc44

16 files changed

+87
-59
lines changed

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,12 @@
3939
to `null` will now result in an `ArgumentNullException` being thrown.
4040
([#5434](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5434))
4141

42+
* Introduced experimental support for automatically retrying export to the otlp
43+
endpoint when transient network errors occur. Users can enable this feature by
44+
setting `OTEL_DOTNET_EXPERIMENTAL_OTLP_ENABLE_INMEMORY_RETRY` environment
45+
variable to true.
46+
([#5435](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5435))
47+
4248
## 1.7.0
4349

4450
Released 2023-Dec-08

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ internal sealed class ExperimentalOptions
1616

1717
public const string EmitLogEventEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES";
1818

19+
public const string EnableInMemoryRetryEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_ENABLE_INMEMORY_RETRY";
20+
1921
public ExperimentalOptions()
2022
: this(new ConfigurationBuilder().AddEnvironmentVariables().Build())
2123
{
@@ -27,10 +29,24 @@ public ExperimentalOptions(IConfiguration configuration)
2729
{
2830
this.EmitLogEventAttributes = emitLogEventAttributes;
2931
}
32+
33+
if (configuration.TryGetBoolValue(EnableInMemoryRetryEnvVar, out var enableInMemoryRetry))
34+
{
35+
this.EnableInMemoryRetry = enableInMemoryRetry;
36+
}
3037
}
3138

3239
/// <summary>
33-
/// Gets or sets a value indicating whether log event attributes should be exported.
40+
/// Gets a value indicating whether log event attributes should be exported.
41+
/// </summary>
42+
public bool EmitLogEventAttributes { get; }
43+
44+
/// <summary>
45+
/// Gets a value indicating whether or not in-memory retry should be enabled for transient errors.
3446
/// </summary>
35-
public bool EmitLogEventAttributes { get; set; } = false;
47+
/// <remarks>
48+
/// Specification: <see
49+
/// href="https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry"/>.
50+
/// </remarks>
51+
public bool EnableInMemoryRetry { get; }
3652
}

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptions.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
using Microsoft.Extensions.Configuration;
1212
using Microsoft.Extensions.DependencyInjection;
1313
using Microsoft.Extensions.Options;
14+
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
1415
using OpenTelemetry.Internal;
1516
using OpenTelemetry.Trace;
1617

@@ -225,6 +226,7 @@ public Func<HttpClient> HttpClientFactory
225226
internal static void RegisterOtlpExporterOptionsFactory(IServiceCollection services)
226227
{
227228
services.RegisterOptionsFactory(CreateOtlpExporterOptions);
229+
services.RegisterOptionsFactory(configuration => new ExperimentalOptions(configuration));
228230
}
229231

230232
internal static OtlpExporterOptions CreateOtlpExporterOptions(

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#endif
77
using System.Reflection;
88
using Grpc.Core;
9+
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
910
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
1011
#if NETSTANDARD2_1 || NET6_0_OR_GREATER
1112
using Grpc.Net.Client;
@@ -88,7 +89,7 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
8889
return headers;
8990
}
9091

91-
public static OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportTransmissionHandler(this OtlpExporterOptions options)
92+
public static OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
9293
{
9394
var exportClient = GetTraceExportClient(options);
9495

@@ -99,10 +100,12 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
99100
? httpTraceExportClient.HttpClient.Timeout.TotalMilliseconds
100101
: options.TimeoutMilliseconds;
101102

102-
return new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
103+
return experimentalOptions.EnableInMemoryRetry
104+
? new OtlpExporterRetryTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds)
105+
: new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
103106
}
104107

105-
public static OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest> GetMetricsExportTransmissionHandler(this OtlpExporterOptions options)
108+
public static OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest> GetMetricsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
106109
{
107110
var exportClient = GetMetricsExportClient(options);
108111

@@ -113,21 +116,21 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
113116
? httpMetricsExportClient.HttpClient.Timeout.TotalMilliseconds
114117
: options.TimeoutMilliseconds;
115118

116-
return new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
119+
return experimentalOptions.EnableInMemoryRetry
120+
? new OtlpExporterRetryTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds)
121+
: new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
117122
}
118123

119-
public static OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest> GetLogsExportTransmissionHandler(this OtlpExporterOptions options)
124+
public static OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest> GetLogsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
120125
{
121126
var exportClient = GetLogExportClient(options);
122-
123-
// `HttpClient.Timeout.TotalMilliseconds` would be populated with the correct timeout value for both the exporter configuration cases:
124-
// 1. User provides their own HttpClient. This case is straightforward as the user wants to use their `HttpClient` and thereby the same client's timeout value.
125-
// 2. If the user configures timeout via the exporter options, then the timeout set for the `HttpClient` initialized by the exporter will be set to user provided value.
126127
double timeoutMilliseconds = exportClient is OtlpHttpLogExportClient httpLogExportClient
127128
? httpLogExportClient.HttpClient.Timeout.TotalMilliseconds
128129
: options.TimeoutMilliseconds;
129130

130-
return new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
131+
return experimentalOptions.EnableInMemoryRetry
132+
? new OtlpExporterRetryTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds)
133+
: new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
131134
}
132135

133136
public static IExportClient<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportClient(this OtlpExporterOptions options) =>

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ internal OtlpLogExporter(
6262
OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable(key, value);
6363
};
6464

65-
this.transmissionHandler = transmissionHandler ?? exporterOptions.GetLogsExportTransmissionHandler();
65+
this.transmissionHandler = transmissionHandler ?? exporterOptions.GetLogsExportTransmissionHandler(experimentalOptions!);
6666

6767
this.otlpLogRecordTransformer = new OtlpLogRecordTransformer(sdkLimitOptions!, experimentalOptions!);
6868
}

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporterHelperExtensions.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,6 @@ private static void RegisterOptions(IServiceCollection services)
404404
{
405405
OtlpExporterOptions.RegisterOtlpExporterOptionsFactory(services);
406406
services.RegisterOptionsFactory(configuration => new SdkLimitOptions(configuration));
407-
services.RegisterOptionsFactory(configuration => new ExperimentalOptions(configuration));
408407
}
409408

410409
private static T GetOptions<T>(

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporter.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,19 @@ public class OtlpMetricExporter : BaseExporter<Metric>
2525
/// </summary>
2626
/// <param name="options">Configuration options for the exporter.</param>
2727
public OtlpMetricExporter(OtlpExporterOptions options)
28-
: this(options, transmissionHandler: null)
28+
: this(options, experimentalOptions: new(), transmissionHandler: null)
2929
{
3030
}
3131

3232
/// <summary>
3333
/// Initializes a new instance of the <see cref="OtlpMetricExporter"/> class.
3434
/// </summary>
3535
/// <param name="options">Configuration options for the export.</param>
36+
/// <param name="experimentalOptions"><see cref="ExperimentalOptions"/>.</param>
3637
/// <param name="transmissionHandler"><see cref="OtlpExporterTransmissionHandler{T}"/>.</param>
3738
internal OtlpMetricExporter(
3839
OtlpExporterOptions options,
40+
ExperimentalOptions experimentalOptions,
3941
OtlpExporterTransmissionHandler<OtlpCollector.ExportMetricsServiceRequest> transmissionHandler = null)
4042
{
4143
// Each of the Otlp exporters: Traces, Metrics, and Logs set the same value for `OtlpKeyValueTransformer.LogUnsupportedAttributeType`
@@ -50,7 +52,7 @@ internal OtlpMetricExporter(
5052
OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable(key, value);
5153
};
5254

53-
this.transmissionHandler = transmissionHandler ?? options.GetMetricsExportTransmissionHandler();
55+
this.transmissionHandler = transmissionHandler ?? options.GetMetricsExportTransmissionHandler(experimentalOptions);
5456
}
5557

5658
internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource();

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporterExtensions.cs

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using Microsoft.Extensions.DependencyInjection;
88
using Microsoft.Extensions.Options;
99
using OpenTelemetry.Exporter;
10+
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
1011
using OpenTelemetry.Internal;
1112

1213
namespace OpenTelemetry.Metrics;
@@ -98,6 +99,7 @@ public static MeterProviderBuilder AddOtlpExporter(
9899
return BuildOtlpExporterMetricReader(
99100
exporterOptions,
100101
sp.GetRequiredService<IOptionsMonitor<MetricReaderOptions>>().Get(finalOptionsName),
102+
sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
101103
sp);
102104
});
103105
}
@@ -169,19 +171,24 @@ public static MeterProviderBuilder AddOtlpExporter(
169171

170172
configureExporterAndMetricReader?.Invoke(exporterOptions, metricReaderOptions);
171173

172-
return BuildOtlpExporterMetricReader(exporterOptions, metricReaderOptions, sp);
174+
return BuildOtlpExporterMetricReader(
175+
exporterOptions,
176+
metricReaderOptions,
177+
sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
178+
sp);
173179
});
174180
}
175181

176182
internal static MetricReader BuildOtlpExporterMetricReader(
177183
OtlpExporterOptions exporterOptions,
178184
MetricReaderOptions metricReaderOptions,
185+
ExperimentalOptions experimentalOptions,
179186
IServiceProvider serviceProvider,
180187
Func<BaseExporter<Metric>, BaseExporter<Metric>>? configureExporterInstance = null)
181188
{
182189
exporterOptions.TryEnableIHttpClientFactoryIntegration(serviceProvider, "OtlpMetricExporter");
183190

184-
BaseExporter<Metric> metricExporter = new OtlpMetricExporter(exporterOptions);
191+
BaseExporter<Metric> metricExporter = new OtlpMetricExporter(exporterOptions, experimentalOptions);
185192

186193
if (configureExporterInstance != null)
187194
{

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporter.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public class OtlpTraceExporter : BaseExporter<Activity>
2626
/// </summary>
2727
/// <param name="options">Configuration options for the export.</param>
2828
public OtlpTraceExporter(OtlpExporterOptions options)
29-
: this(options, sdkLimitOptions: new(), transmissionHandler: null)
29+
: this(options, sdkLimitOptions: new(), experimentalOptions: new(), transmissionHandler: null)
3030
{
3131
}
3232

@@ -35,10 +35,12 @@ public OtlpTraceExporter(OtlpExporterOptions options)
3535
/// </summary>
3636
/// <param name="exporterOptions"><see cref="OtlpExporterOptions"/>.</param>
3737
/// <param name="sdkLimitOptions"><see cref="SdkLimitOptions"/>.</param>
38+
/// <param name="experimentalOptions"><see cref="ExperimentalOptions"/>.</param>
3839
/// <param name="transmissionHandler"><see cref="OtlpExporterTransmissionHandler{T}"/>.</param>
3940
internal OtlpTraceExporter(
4041
OtlpExporterOptions exporterOptions,
4142
SdkLimitOptions sdkLimitOptions,
43+
ExperimentalOptions experimentalOptions,
4244
OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest> transmissionHandler = null)
4345
{
4446
Debug.Assert(exporterOptions != null, "exporterOptions was null");
@@ -50,7 +52,7 @@ internal OtlpTraceExporter(
5052

5153
ConfigurationExtensions.LogInvalidEnvironmentVariable = OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable;
5254

53-
this.transmissionHandler = transmissionHandler ?? exporterOptions.GetTraceExportTransmissionHandler();
55+
this.transmissionHandler = transmissionHandler ?? exporterOptions.GetTraceExportTransmissionHandler(experimentalOptions);
5456
}
5557

5658
internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource();

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporterHelperExtensions.cs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,19 +92,24 @@ public static TracerProviderBuilder AddOtlpExporter(
9292
// instance.
9393
var sdkOptionsManager = sp.GetRequiredService<IOptionsMonitor<SdkLimitOptions>>().CurrentValue;
9494

95-
return BuildOtlpExporterProcessor(exporterOptions, sdkOptionsManager, sp);
95+
return BuildOtlpExporterProcessor(
96+
exporterOptions,
97+
sdkOptionsManager,
98+
sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
99+
sp);
96100
});
97101
}
98102

99103
internal static BaseProcessor<Activity> BuildOtlpExporterProcessor(
100104
OtlpExporterOptions exporterOptions,
101105
SdkLimitOptions sdkLimitOptions,
106+
ExperimentalOptions experimentalOptions,
102107
IServiceProvider serviceProvider,
103108
Func<BaseExporter<Activity>, BaseExporter<Activity>>? configureExporterInstance = null)
104109
{
105110
exporterOptions.TryEnableIHttpClientFactoryIntegration(serviceProvider, "OtlpTraceExporter");
106111

107-
BaseExporter<Activity> otlpExporter = new OtlpTraceExporter(exporterOptions, sdkLimitOptions);
112+
BaseExporter<Activity> otlpExporter = new OtlpTraceExporter(exporterOptions, sdkLimitOptions, experimentalOptions);
108113

109114
if (configureExporterInstance != null)
110115
{

0 commit comments

Comments
 (0)