Skip to content

Commit d8dea12

Browse files
surbhigarg92surbhigarg92
committed
feat: grpc metrics
1 parent 4435485 commit d8dea12

10 files changed

+189
-55
lines changed

‎google-cloud-spanner/pom.xml

+4
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,10 @@
191191
<groupId>io.grpc</groupId>
192192
<artifactId>grpc-stub</artifactId>
193193
</dependency>
194+
<dependency>
195+
<groupId>io.grpc</groupId>
196+
<artifactId>grpc-opentelemetry</artifactId>
197+
</dependency>
194198
<dependency>
195199
<groupId>com.google.api</groupId>
196200
<artifactId>api-common</artifactId>

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java

+48-6
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,13 @@
2121
import com.google.common.collect.ImmutableList;
2222
import com.google.common.collect.ImmutableMap;
2323
import com.google.common.collect.ImmutableSet;
24+
import com.google.common.collect.Iterables;
2425
import io.opentelemetry.api.common.AttributeKey;
2526
import io.opentelemetry.sdk.metrics.Aggregation;
2627
import io.opentelemetry.sdk.metrics.InstrumentSelector;
2728
import io.opentelemetry.sdk.metrics.InstrumentType;
2829
import io.opentelemetry.sdk.metrics.View;
30+
import java.util.Collection;
2931
import java.util.Map;
3032
import java.util.Set;
3133
import java.util.stream.Collectors;
@@ -36,6 +38,7 @@ public class BuiltInMetricsConstant {
3638
public static final String METER_NAME = "spanner.googleapis.com/internal/client";
3739
public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME;
3840
static final String SPANNER_METER_NAME = "spanner-java";
41+
static final String GRPC_METER_NAME = "grpc-java";
3942
static final String GFE_LATENCIES_NAME = "gfe_latencies";
4043
static final String OPERATION_LATENCIES_NAME = "operation_latencies";
4144
static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies";
@@ -66,12 +69,7 @@ public class BuiltInMetricsConstant {
6669

6770
// These metric labels will be promoted to the spanner monitored resource fields
6871
public static final Set<AttributeKey<String>> SPANNER_PROMOTED_RESOURCE_LABELS =
69-
ImmutableSet.of(
70-
PROJECT_ID_KEY,
71-
INSTANCE_ID_KEY,
72-
INSTANCE_CONFIG_ID_KEY,
73-
LOCATION_ID_KEY,
74-
CLIENT_HASH_KEY);
72+
ImmutableSet.of(INSTANCE_ID_KEY);
7573

7674
public static final AttributeKey<String> DATABASE_KEY = AttributeKey.stringKey("database");
7775
public static final AttributeKey<String> CLIENT_UID_KEY = AttributeKey.stringKey("client_uid");
@@ -102,6 +100,11 @@ public class BuiltInMetricsConstant {
102100
DIRECT_PATH_ENABLED_KEY,
103101
DIRECT_PATH_USED_KEY);
104102

103+
public static final Set<String> GRPC_ATTRIBUTES =
104+
ImmutableSet.of(
105+
"grpc_lb_rls_data_plane_target",
106+
"grpc_lb_pick_result");
107+
105108
static Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM =
106109
Aggregation.explicitBucketHistogram(
107110
ImmutableList.of(
@@ -111,6 +114,21 @@ public class BuiltInMetricsConstant {
111114
10000.0, 20000.0, 50000.0, 100000.0, 200000.0, 400000.0, 800000.0, 1600000.0,
112115
3200000.0));
113116

117+
static final Collection<String> GRPC_METRICS_TO_ENABLE =
118+
ImmutableList.of(
119+
"grpc.lb.rls.default_target_picks",
120+
"grpc.lb.rls.target_picks",
121+
"grpc.xds_client.server_failure",
122+
"grpc.xds_client.resource_updates_invalid");
123+
124+
static final Collection<String> GRPC_METRICS_ENABLED_BY_DEFAULT =
125+
ImmutableList.of(
126+
"grpc.client.attempt.sent_total_compressed_message_size",
127+
"grpc.client.attempt.rcvd_total_compressed_message_size",
128+
"grpc.client.attempt.started",
129+
"grpc.client.attempt.duration",
130+
"grpc.client.call.duration");
131+
114132
static Map<InstrumentSelector, View> getAllViews() {
115133
ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
116134
defineView(
@@ -153,6 +171,7 @@ static Map<InstrumentSelector, View> getAllViews() {
153171
Aggregation.sum(),
154172
InstrumentType.COUNTER,
155173
"1");
174+
defineGRPCView(views);
156175
return views.build();
157176
}
158177

@@ -183,4 +202,27 @@ private static void defineView(
183202
.build();
184203
viewMap.put(selector, view);
185204
}
205+
206+
private static void defineGRPCView(ImmutableMap.Builder<InstrumentSelector, View> viewMap) {
207+
for (String metric :
208+
ImmutableList.copyOf(Iterables.concat(BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE, BuiltInMetricsConstant.GRPC_METRICS_ENABLED_BY_DEFAULT))) {
209+
InstrumentSelector selector =
210+
InstrumentSelector.builder()
211+
.setName(metric)
212+
.setMeterName(BuiltInMetricsConstant.GRPC_METER_NAME)
213+
.build();
214+
Set<String> attributesFilter =
215+
BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream()
216+
.map(AttributeKey::getKey)
217+
.collect(Collectors.toSet());
218+
219+
attributesFilter.addAll(BuiltInMetricsConstant.GRPC_ATTRIBUTES);
220+
View view =
221+
View.builder()
222+
.setName(BuiltInMetricsConstant.METER_NAME + '/' + metric.replace(".", "/"))
223+
.setAttributeFilter(attributesFilter)
224+
.build();
225+
viewMap.put(selector, view);
226+
}
227+
}
186228
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java

+53-8
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,28 @@
2424
import static com.google.cloud.spanner.BuiltInMetricsConstant.LOCATION_ID_KEY;
2525
import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY;
2626

27+
import com.google.api.core.ApiFunction;
28+
import com.google.api.gax.core.GaxProperties;
29+
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
2730
import com.google.auth.Credentials;
2831
import com.google.cloud.opentelemetry.detection.AttributeKeys;
2932
import com.google.cloud.opentelemetry.detection.DetectedPlatform;
3033
import com.google.cloud.opentelemetry.detection.GCPPlatformDetector;
34+
import com.google.common.collect.ImmutableList;
35+
import com.google.common.collect.Iterables;
3136
import com.google.common.hash.HashFunction;
3237
import com.google.common.hash.Hashing;
38+
import io.grpc.ManagedChannelBuilder;
39+
import io.grpc.opentelemetry.GrpcOpenTelemetry;
3340
import io.opentelemetry.api.OpenTelemetry;
41+
import io.opentelemetry.api.common.Attributes;
42+
import io.opentelemetry.api.common.AttributesBuilder;
3443
import io.opentelemetry.sdk.OpenTelemetrySdk;
44+
import io.opentelemetry.sdk.metrics.InstrumentSelector;
3545
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
3646
import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
47+
import io.opentelemetry.sdk.metrics.View;
48+
import io.opentelemetry.sdk.resources.Resource;
3749
import java.io.IOException;
3850
import java.lang.management.ManagementFactory;
3951
import java.lang.reflect.Method;
@@ -66,6 +78,9 @@ OpenTelemetry getOrCreateOpenTelemetry(
6678
BuiltInMetricsView.registerBuiltinMetrics(
6779
SpannerCloudMonitoringExporter.create(projectId, credentials, monitoringHost),
6880
sdkMeterProviderBuilder);
81+
82+
83+
sdkMeterProviderBuilder.setResource(Resource.create(createResourceAttributes(projectId)));
6984
SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build();
7085
this.openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
7186
Runtime.getRuntime().addShutdownHook(new Thread(sdkMeterProvider::close));
@@ -80,15 +95,45 @@ OpenTelemetry getOrCreateOpenTelemetry(
8095
}
8196
}
8297

83-
Map<String, String> createClientAttributes(String projectId, String client_name) {
98+
public void enableGrpcMetrics(
99+
InstantiatingGrpcChannelProvider.Builder channelProviderBuilder,
100+
String projectId,
101+
@Nullable Credentials credentials,
102+
@Nullable String monitoringHost) {
103+
GrpcOpenTelemetry grpcOpenTelemetry =
104+
GrpcOpenTelemetry.newBuilder()
105+
.sdk(this.getOrCreateOpenTelemetry(projectId, credentials, monitoringHost))
106+
.enableMetrics(BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE)
107+
// .disableMetrics(BuiltInMetricsConstant.GRPC_METRICS_ENABLED_BY_DEFAULT)
108+
.build();
109+
ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> channelConfigurator =
110+
channelProviderBuilder.getChannelConfigurator();
111+
channelProviderBuilder.setChannelConfigurator(
112+
b -> {
113+
grpcOpenTelemetry.configureChannelBuilder(b);
114+
if (channelConfigurator != null) {
115+
return channelConfigurator.apply(b);
116+
}
117+
return b;
118+
});
119+
}
120+
121+
Attributes createResourceAttributes(String projectId) {
122+
AttributesBuilder attributesBuilder =
123+
Attributes.builder()
124+
.put(PROJECT_ID_KEY.getKey(), projectId)
125+
.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown")
126+
.put(CLIENT_HASH_KEY.getKey(), generateClientHash(getDefaultTaskValue()))
127+
.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
128+
129+
return attributesBuilder.build();
130+
}
131+
132+
Map<String, String> createClientAttributes() {
84133
Map<String, String> clientAttributes = new HashMap<>();
85-
clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
86-
clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId);
87-
clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown");
88-
clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name);
89-
String clientUid = getDefaultTaskValue();
90-
clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid);
91-
clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid));
134+
clientAttributes.put(
135+
CLIENT_NAME_KEY.getKey(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()));
136+
clientAttributes.put(CLIENT_UID_KEY.getKey(), getDefaultTaskValue());
92137
return clientAttributes;
93138
}
94139

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java

+19-8
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import io.opentelemetry.sdk.metrics.data.MetricData;
4242
import io.opentelemetry.sdk.metrics.data.PointData;
4343
import io.opentelemetry.sdk.metrics.export.MetricExporter;
44+
import io.opentelemetry.sdk.resources.Resource;
4445
import java.io.IOException;
4546
import java.time.Duration;
4647
import java.util.ArrayList;
@@ -104,6 +105,18 @@ static SpannerCloudMonitoringExporter create(
104105

105106
@Override
106107
public CompletableResultCode export(@Nonnull Collection<MetricData> collection) {
108+
// Print
109+
collection.stream()
110+
.forEach(md -> {
111+
System.out.println("Name: " + md.getName()); // Print the name
112+
113+
md.getData().getPoints().forEach(point -> {
114+
System.out.println("Attributes: " + point.getAttributes()); // Print attributes for each point
115+
});
116+
117+
System.out.println("----------------------"); // Separator for readability
118+
});
119+
107120
if (client.isShutdown()) {
108121
logger.log(Level.WARNING, "Exporter is shut down");
109122
return CompletableResultCode.ofFailure();
@@ -123,7 +136,7 @@ private CompletableResultCode exportSpannerClientMetrics(Collection<MetricData>
123136
// Log warnings for metrics that will be skipped.
124137
boolean mustFilter = false;
125138
if (spannerMetricData.stream()
126-
.flatMap(metricData -> metricData.getData().getPoints().stream())
139+
.map(metricData -> metricData.getResource())
127140
.anyMatch(this::shouldSkipPointDataDueToProjectId)) {
128141
logger.log(
129142
Level.WARNING, "Some metric data contain a different projectId. These will be skipped.");
@@ -198,15 +211,13 @@ public void onSuccess(List<Empty> empty) {
198211
}
199212

200213
private boolean shouldSkipMetricData(MetricData metricData) {
201-
return metricData.getData().getPoints().stream()
202-
.anyMatch(
203-
pd ->
204-
shouldSkipPointDataDueToProjectId(pd)
205-
|| shouldSkipPointDataDueToMissingInstanceId(pd));
214+
return shouldSkipPointDataDueToProjectId(metricData.getResource())
215+
|| metricData.getData().getPoints().stream()
216+
.anyMatch(pd -> shouldSkipPointDataDueToMissingInstanceId(pd));
206217
}
207218

208-
private boolean shouldSkipPointDataDueToProjectId(PointData pointData) {
209-
return !spannerProjectId.equals(SpannerCloudMonitoringExporterUtils.getProjectId(pointData));
219+
private boolean shouldSkipPointDataDueToProjectId(Resource resource) {
220+
return !spannerProjectId.equals(SpannerCloudMonitoringExporterUtils.getProjectId(resource));
210221
}
211222

212223
private boolean shouldSkipPointDataDueToMissingInstanceId(PointData pointData) {

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java

+25-7
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import static com.google.api.MetricDescriptor.ValueType.DOUBLE;
2424
import static com.google.api.MetricDescriptor.ValueType.INT64;
2525
import static com.google.cloud.spanner.BuiltInMetricsConstant.GAX_METER_NAME;
26+
import static com.google.cloud.spanner.BuiltInMetricsConstant.GRPC_METER_NAME;
2627
import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_ID_KEY;
2728
import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY;
2829
import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_METER_NAME;
@@ -52,6 +53,7 @@
5253
import io.opentelemetry.sdk.metrics.data.MetricDataType;
5354
import io.opentelemetry.sdk.metrics.data.PointData;
5455
import io.opentelemetry.sdk.metrics.data.SumData;
56+
import io.opentelemetry.sdk.resources.Resource;
5557
import java.util.ArrayList;
5658
import java.util.List;
5759
import java.util.logging.Level;
@@ -64,8 +66,8 @@ class SpannerCloudMonitoringExporterUtils {
6466

6567
private SpannerCloudMonitoringExporterUtils() {}
6668

67-
static String getProjectId(PointData pointData) {
68-
return pointData.getAttributes().get(PROJECT_ID_KEY);
69+
static String getProjectId(Resource resource) {
70+
return resource.getAttributes().get(PROJECT_ID_KEY);
6971
}
7072

7173
static String getInstanceId(PointData pointData) {
@@ -78,29 +80,43 @@ static List<TimeSeries> convertToSpannerTimeSeries(List<MetricData> collection)
7880
for (MetricData metricData : collection) {
7981
// Get metrics data from GAX library and Spanner library
8082
if (!(metricData.getInstrumentationScopeInfo().getName().equals(GAX_METER_NAME)
81-
|| metricData.getInstrumentationScopeInfo().getName().equals(SPANNER_METER_NAME))) {
83+
|| metricData.getInstrumentationScopeInfo().getName().equals(SPANNER_METER_NAME)
84+
|| metricData.getInstrumentationScopeInfo().getName().equals(GRPC_METER_NAME))) {
8285
// Filter out metric data for instruments that are not part of the spanner metrics list
8386
continue;
8487
}
88+
89+
// Create MonitoredResource Builder
90+
MonitoredResource.Builder monitoredResourceBuilder =
91+
MonitoredResource.newBuilder().setType(SPANNER_RESOURCE_TYPE);
92+
93+
Attributes resourceAttributes = metricData.getResource().getAttributes();
94+
for (AttributeKey<?> key : resourceAttributes.asMap().keySet()) {
95+
monitoredResourceBuilder.putLabels(
96+
key.getKey(), String.valueOf(resourceAttributes.get(key)));
97+
}
98+
8599
metricData.getData().getPoints().stream()
86-
.map(pointData -> convertPointToSpannerTimeSeries(metricData, pointData))
100+
.map(
101+
pointData ->
102+
convertPointToSpannerTimeSeries(metricData, pointData, monitoredResourceBuilder))
87103
.forEach(allTimeSeries::add);
88104
}
89105

90106
return allTimeSeries;
91107
}
92108

93109
private static TimeSeries convertPointToSpannerTimeSeries(
94-
MetricData metricData, PointData pointData) {
110+
MetricData metricData,
111+
PointData pointData,
112+
MonitoredResource.Builder monitoredResourceBuilder) {
95113
TimeSeries.Builder builder =
96114
TimeSeries.newBuilder()
97115
.setMetricKind(convertMetricKind(metricData))
98116
.setValueType(convertValueType(metricData.getType()));
99117
Metric.Builder metricBuilder = Metric.newBuilder().setType(metricData.getName());
100118

101119
Attributes attributes = pointData.getAttributes();
102-
MonitoredResource.Builder monitoredResourceBuilder =
103-
MonitoredResource.newBuilder().setType(SPANNER_RESOURCE_TYPE);
104120

105121
for (AttributeKey<?> key : attributes.asMap().keySet()) {
106122
if (SPANNER_PROMOTED_RESOURCE_LABELS.contains(key)) {
@@ -110,6 +126,8 @@ private static TimeSeries convertPointToSpannerTimeSeries(
110126
}
111127
}
112128

129+
metricBuilder.putAllLabels(BuiltInMetricsProvider.INSTANCE.createClientAttributes());
130+
113131
builder.setResource(monitoredResourceBuilder.build());
114132
builder.setMetric(metricBuilder.build());
115133

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java

+7-2
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.google.api.gax.core.GaxProperties;
2828
import com.google.api.gax.grpc.GrpcCallContext;
2929
import com.google.api.gax.grpc.GrpcInterceptorProvider;
30+
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
3031
import com.google.api.gax.longrunning.OperationTimedPollAlgorithm;
3132
import com.google.api.gax.retrying.RetrySettings;
3233
import com.google.api.gax.rpc.ApiCallContext;
@@ -1971,6 +1972,11 @@ public ApiTracerFactory getApiTracerFactory() {
19711972
return createApiTracerFactory(false, false);
19721973
}
19731974

1975+
public void enablegRPCMetrics(InstantiatingGrpcChannelProvider.Builder channelProviderBuilder) {
1976+
this.builtInMetricsProvider.enableGrpcMetrics(
1977+
channelProviderBuilder, this.getProjectId(), getCredentials(), this.monitoringHost);
1978+
}
1979+
19741980
public ApiTracerFactory getApiTracerFactory(boolean isAdminClient, boolean isEmulatorEnabled) {
19751981
return createApiTracerFactory(isAdminClient, isEmulatorEnabled);
19761982
}
@@ -2018,8 +2024,7 @@ private ApiTracerFactory createMetricsApiTracerFactory() {
20182024
return openTelemetry != null
20192025
? new BuiltInMetricsTracerFactory(
20202026
new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME),
2021-
builtInMetricsProvider.createClientAttributes(
2022-
this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass())))
2027+
new HashMap<>())
20232028
: null;
20242029
}
20252030

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java

+5
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,11 @@ public GapicSpannerRpc(final SpannerOptions options) {
370370
defaultChannelProviderBuilder.setAttemptDirectPath(true);
371371
defaultChannelProviderBuilder.setAttemptDirectPathXds();
372372
}
373+
374+
// Use condition to enable gRPC metrics
375+
if (true) {
376+
options.enablegRPCMetrics(defaultChannelProviderBuilder);
377+
}
373378
if (options.isUseVirtualThreads()) {
374379
ExecutorService executor =
375380
tryCreateVirtualThreadPerTaskExecutor("spanner-virtual-grpc-executor");

0 commit comments

Comments
 (0)