[POC] Add some metrics to debug at scale

pkg/metrics/client_go_adapter.go

@@ -18,17 +18,34 @@ package metrics
 
 import (
 	"context"
+	"math"
+	"net/url"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	clientmetrics "k8s.io/client-go/tools/metrics"
 )
 
-// this file contains setup logic to initialize the myriad of places
-// that client-go registers metrics.  We copy the names and formats
-// from Kubernetes so that we match the core controllers.
-
 var (
-	// client metrics.
+	// requestLatency is a Prometheus Histogram metric type partitioned by
+	// "verb", and "host" labels. It is used for the rest client latency metrics.
+	requestLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "rest_client_request_duration_seconds",
+			Help:    "Request latency in seconds. Broken down by verb, and host.",
+			Buckets: []float64{math.Inf(+1)}, // Intentionally using minimum buckets for better performance / lower cardinality.
+		},
+		[]string{"verb", "host"},
+	)
+
+	rateLimiterLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "rest_client_rate_limiter_duration_seconds",
+			Help:    "Client side rate limiter latency in seconds. Broken down by verb, and host.",
+			Buckets: []float64{math.Inf(+1)}, // Intentionally using minimum buckets for better performance / lower cardinality.
+		},
+		[]string{"verb", "host"},
+	)
 
 	requestResult = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
@@ -37,6 +54,14 @@ var (
 		},
 		[]string{"code", "method", "host"},
 	)
+
+	requestRetry = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "rest_client_request_retries_total",
+			Help: "Number of request retries, partitioned by status code, verb, and host.",
+		},
+		[]string{"code", "verb", "host"},
+	)
 )
 
 func init() {
@@ -46,11 +71,17 @@ func init() {
 // registerClientMetrics sets up the client latency metrics from client-go.
 func registerClientMetrics() {
 	// register the metrics with our registry
+	Registry.MustRegister(requestLatency)
+	Registry.MustRegister(rateLimiterLatency)
 	Registry.MustRegister(requestResult)
+	Registry.MustRegister(requestRetry)
 
 	// register the metrics with client-go
 	clientmetrics.Register(clientmetrics.RegisterOpts{
-		RequestResult: &resultAdapter{metric: requestResult},
+		RequestLatency:     &LatencyAdapter{metric: requestLatency},
+		RateLimiterLatency: &LatencyAdapter{metric: rateLimiterLatency},
+		RequestResult:      &resultAdapter{metric: requestResult},
+		RequestRetry:       &retryAdapter{requestRetry},
 	})
 }
 
@@ -62,10 +93,28 @@ func registerClientMetrics() {
 // copied (more-or-less directly) from k8s.io/kubernetes setup code
 // (which isn't anywhere in an easily-importable place).
 
+// LatencyAdapter implements LatencyMetric.
+type LatencyAdapter struct {
+	metric *prometheus.HistogramVec
+}
+
+// Observe increments the request latency metric for the given verb/URL.
+func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) {
+	l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds())
+}
+
 type resultAdapter struct {
 	metric *prometheus.CounterVec
 }
 
 func (r *resultAdapter) Increment(_ context.Context, code, method, host string) {
 	r.metric.WithLabelValues(code, method, host).Inc()
 }
+
+type retryAdapter struct {
+	metric *prometheus.CounterVec
+}
+
+func (r *retryAdapter) IncrementRetry(_ context.Context, code, method, host string) {
+	r.metric.WithLabelValues(code, method, host).Inc()
+}