Main import duration metric (#149)

* add metric for main import flow duration * bump version
Concordium · Dec 1, 2023 · afb60d3 · afb60d3
1 parent d30ab26
commit afb60d3
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 43 deletions.
diff --git a/backend/Application/Aggregates/Contract/ContractAggregate.cs b/backend/Application/Aggregates/Contract/ContractAggregate.cs
@@ -517,7 +517,7 @@ private async Task NodeImportRange(IContractNodeClient client, ulong fromBlockHe
         for (var height = fromBlockHeight; height <= toBlockHeight; height++)
         {
             using var __ = TraceContext.StartActivity(NodeImportJobLoopActivity);
-            using var durationMetric = new ContractMetrics.DurationMetric(ImportSource.NodeImport);
+            using var durationMetric = ContractMetrics.CreateContractReadDurationMetric(ImportSource.NodeImport);
             try
             {
                 await using var repository = await _repositoryFactory.CreateContractRepositoryAsync();

diff --git a/backend/Application/Aggregates/Contract/Jobs/InitialContractAggregateCatchUpJob.cs b/backend/Application/Aggregates/Contract/Jobs/InitialContractAggregateCatchUpJob.cs
@@ -68,7 +68,7 @@ public async Task<ulong> BatchImportJob(ulong heightFrom, ulong heightTo, Cancel
         return await Policies.GetTransientPolicy<ulong>(GetUniqueIdentifier(), _logger, _contractAggregateOptions.RetryCount, _contractAggregateOptions.RetryDelay)
             .ExecuteAsync(async () =>
             {
-                using var durationMetric = new ContractMetrics.DurationMetric(ImportSource.DatabaseImport);
+                using var durationMetric =ContractMetrics.CreateContractReadDurationMetric(ImportSource.DatabaseImport);
                 try
                 {
                     await using var repository = await _repositoryFactory.CreateContractRepositoryAsync();

diff --git a/backend/Application/Aggregates/Contract/Observability/ContractMetrics.cs b/backend/Application/Aggregates/Contract/Observability/ContractMetrics.cs
@@ -1,22 +1,11 @@
-using System.Diagnostics;
 using Application.Aggregates.Contract.Types;
 using Application.Observability;
 using Prometheus;
-using static Application.Observability.ApplicationMetrics;
 
 namespace Application.Aggregates.Contract.Observability;
 
 internal static class ContractMetrics
 {
-    private static readonly Histogram ReadDuration = Metrics.CreateHistogram(
-        "contract_read_duration_seconds",
-        "Duration of import in seconds",
-        new HistogramConfiguration
-        {
-            LabelNames = new[] { "source", "exception" }
-        }
-    );
-
     private static readonly Counter ImportedTransactionEvents = Metrics.CreateCounter(
         "contract_imported_transaction_events_total", "Number of transaction event which has been processed and triggered one or more events",
         new CounterConfiguration
@@ -36,31 +25,8 @@ internal static void SetReadHeight(double value, ImportSource source)
     {
         ApplicationMetrics.SetReadHeight(value, "contract", source);
     }
-
-    internal class DurationMetric : IDisposable
-    {
-        private string _exceptionName = "";
-        private readonly Stopwatch _time;
-        private readonly ImportSource _source;
-
-        public DurationMetric(ImportSource source)
-        {
-            _source = source;
-            _time = Stopwatch.StartNew();
-        }
 
-        internal void SetException(Exception ex)
-        {
-            _exceptionName = PrettyPrintException(ex);
-        }
-
-        public void Dispose()
-        {
-            var elapsedSeconds = _time.ElapsedMilliseconds / 1_000d;
-            ReadDuration
-                .WithLabels(_source.ToStringCached(), _exceptionName)
-                .Observe(elapsedSeconds);
-        }
-    }
+    internal static ApplicationMetrics.DurationMetric CreateContractReadDurationMetric(ImportSource source) => 
+        new("contract_import", source);
 }
 
diff --git a/backend/Application/Api/GraphQL/Import/ImportWriteController.cs b/backend/Application/Api/GraphQL/Import/ImportWriteController.cs
@@ -158,7 +158,7 @@ private async Task ReadAndPublishInitialState()
     private async Task<BlockWriteResult> WriteData(BlockDataPayload payload, ConsensusInfo consensusStatus, CancellationToken stoppingToken)
     {
         using var counter = _metrics.MeasureDuration(nameof(ImportWriteController), nameof(WriteData));
-
+        using var durationMetric = new ApplicationMetrics.DurationMetric("main_import", ImportSource.NodeImport);
         var txScope = CreateTransactionScope();
 
         BlockWriteResult result;
@@ -167,7 +167,7 @@ private async Task<BlockWriteResult> WriteData(BlockDataPayload payload, Consens
             var importState = payload switch
             {
                 GenesisBlockDataPayload genesisPayload => ImportState.CreateGenesisState(
-                    genesisPayload, 
+                    genesisPayload,
                     (int)consensusStatus.EpochDuration.TotalMilliseconds
                 ),
                 _ => await _importStateController.GetState()
@@ -184,6 +184,11 @@ private async Task<BlockWriteResult> WriteData(BlockDataPayload payload, Consens
 
             txScope.Complete();
         }
+        catch (Exception exception)
+        {
+            durationMetric.SetException(exception);
+            throw;
+        }
         finally
         {
             using (_metrics.MeasureDuration(nameof(ImportWriteController), "WriteDataDisposeTx"))

diff --git a/backend/Application/Application.csproj b/backend/Application/Application.csproj
@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <Nullable>enable</Nullable>
         <ImplicitUsings>disable</ImplicitUsings>
-        <Version>1.8.1</Version>
+        <Version>1.8.2</Version>
         <IsWindows Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindows>
         <IsOSX Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsOSX>
         <IsLinux Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinux>

diff --git a/backend/Application/Observability/ApplicationMetrics.cs b/backend/Application/Observability/ApplicationMetrics.cs
@@ -10,6 +10,15 @@ namespace Application.Observability;
 
 internal static class ApplicationMetrics
 {
+    private static readonly Histogram ProcessDuration = Metrics.CreateHistogram(
+        "process_duration_seconds",
+        "Duration of a process in seconds",
+        new HistogramConfiguration
+        {
+            LabelNames = new[] { "process", "source", "exception" }
+        }
+    );
+
     private static readonly Gauge ProcessReadHeight = Metrics.CreateGauge(
         "import_process_read_height",
         "Max height read by an import process",
@@ -43,6 +52,15 @@ internal static class ApplicationMetrics
             LabelNames = new[] { "process", "exception" }
         }
     );
+
+    private static void AddProcessDuration(TimeSpan elapsed, string process, ImportSource source, Exception? exception)
+    {
+        var exceptionName = exception != null ? PrettyPrintException(exception) : "";
+        var elapsedSeconds = elapsed.TotalMilliseconds / 1_000d;
+        ProcessDuration
+            .WithLabels(process, source.ToStringCached(), exceptionName)
+            .Observe(elapsedSeconds);
+    }
 
     internal static void SetReadHeight(double value, string processIdentifier, ImportSource source)
     {
@@ -65,6 +83,31 @@ internal static void IncRetryPolicyExceptions(string process, Exception exceptio
             .Inc();
     }
 
+    internal class DurationMetric : IDisposable
+    {
+        private Exception? _exception;
+        private readonly Stopwatch _time;
+        private readonly ImportSource _source;
+        private readonly string _process;
+
+        public DurationMetric(string process, ImportSource source)
+        {
+            _process = process;
+            _source = source;
+            _time = Stopwatch.StartNew();
+        }
+
+        internal void SetException(Exception ex)
+        {
+            _exception = ex;
+        }
+
+        public void Dispose()
+        {
+            AddProcessDuration(_time.Elapsed, _process, _source, _exception);
+        }
+    }
+
     internal class GraphQlDurationMetric : IDisposable
     {
         /// <summary>
@@ -155,8 +198,8 @@ private string CreateOperationDisplayName()
             }
         }
     }
-    
-    internal static string PrettyPrintException(Exception ex)
+
+    private static string PrettyPrintException(Exception ex)
     {
         var type = ex.GetType();
         if (type.GenericTypeArguments.Length == 0)