[server] Metric Name fix and math fix for HeartbeatMonitor (#869)

* [server] Metric Name fix and math fix for HeartbeatMonitor The resulting metric name contained an extra set of '--' when being hooked up to the reporter. Additionally, the metric was reporting the time of the last heartbeat, but not the time SINCE the last heartbeat.
linkedin · Feb 21, 2024 · c29bd65 · c29bd65
1 parent 7db277b
commit c29bd65
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 9 deletions.
diff --git a/.../main/java/com/linkedin/davinci/stats/ingestion/heartbeat/HeartbeatMonitoringService.java b/.../main/java/com/linkedin/davinci/stats/ingestion/heartbeat/HeartbeatMonitoringService.java
@@ -230,7 +230,7 @@ protected void record() {
 
   @FunctionalInterface
   interface ReportLagFunction {
-    void apply(String storeName, int version, String region, double lag);
+    void apply(String storeName, int version, String region, long lag);
   }
 
   private class HeartbeatReporterThread extends Thread {

diff --git a/...ci-client/src/main/java/com/linkedin/davinci/stats/ingestion/heartbeat/HeartbeatStat.java b/...ci-client/src/main/java/com/linkedin/davinci/stats/ingestion/heartbeat/HeartbeatStat.java
@@ -29,12 +29,14 @@ public HeartbeatStat(MetricConfig metricConfig, Set<String> regions) {
     defaultSensor = new WritePathLatencySensor(localRepository, metricConfig, "default-");
   }
 
-  public void recordLeaderLag(String region, double lag) {
-    leaderSensors.computeIfAbsent(region, k -> defaultSensor).record(lag, System.currentTimeMillis());
+  public void recordLeaderLag(String region, long startTime) {
+    long endTime = System.currentTimeMillis();
+    leaderSensors.computeIfAbsent(region, k -> defaultSensor).record(endTime - startTime, endTime);
   }
 
-  public void recordFollowerLag(String region, double lag) {
-    followerSensors.computeIfAbsent(region, k -> defaultSensor).record(lag, System.currentTimeMillis());
+  public void recordFollowerLag(String region, long startTime) {
+    long endTime = System.currentTimeMillis();
+    followerSensors.computeIfAbsent(region, k -> defaultSensor).record(endTime - startTime, endTime);
   }
 
   public WritePathLatencySensor getLeaderLag(String region) {

diff --git a/...t/src/main/java/com/linkedin/davinci/stats/ingestion/heartbeat/HeartbeatStatReporter.java b/...t/src/main/java/com/linkedin/davinci/stats/ingestion/heartbeat/HeartbeatStatReporter.java
@@ -7,8 +7,8 @@
 
 
 public class HeartbeatStatReporter extends AbstractVeniceStatsReporter<HeartbeatStat> {
-  private static final String LEADER_METRIC_PREFIX = "--heartbeat_delay_leader-";
-  private static final String FOLLOWER_METRIC_PREFIX = "--heartbeat_delay_follower-";
+  private static final String LEADER_METRIC_PREFIX = "heartbeat_delay_leader-";
+  private static final String FOLLOWER_METRIC_PREFIX = "heartbeat_delay_follower-";
   private static final String MAX = ".Max";
   private static final String AVG = ".Avg";
 

diff --git a/...src/main/java/com/linkedin/davinci/stats/ingestion/heartbeat/HeartbeatVersionedStats.java b/...src/main/java/com/linkedin/davinci/stats/ingestion/heartbeat/HeartbeatVersionedStats.java
@@ -16,11 +16,11 @@ public HeartbeatVersionedStats(
     super(metricsRepository, metadataRepository, statsInitiator, reporterSupplier, true);
   }
 
-  public void recordLeaderLag(String storeName, int version, String region, double lag) {
+  public void recordLeaderLag(String storeName, int version, String region, long lag) {
     getStats(storeName, version).recordLeaderLag(region, lag);
   }
 
-  public void recordFollowerLag(String storeName, int version, String region, double lag) {
+  public void recordFollowerLag(String storeName, int version, String region, long lag) {
     getStats(storeName, version).recordFollowerLag(region, lag);
   }
 }