feat: enhancing loki logging capture (#1310)

Author: rodrigo-o

Makefile

@@ -166,14 +166,26 @@ test-iex:
 checkpoint-sync: compile-all
 	iex -S mix run -- --checkpoint-sync-url https://mainnet-checkpoint-sync.stakely.io/ --metrics
 
+#▶️ checkpoint-sync.logfile: @ Run an interactive terminal using checkpoint sync with a log file.
+checkpoint-sync.logfile: compile-all
+	iex -S mix run -- --checkpoint-sync-url https://mainnet-checkpoint-sync.stakely.io/ --metrics --log-file ./logs/mainnet.log
+
 #▶️ sepolia: @ Run an interactive terminal using sepolia network
 sepolia: compile-all
 	iex -S mix run -- --checkpoint-sync-url https://sepolia.beaconstate.info --network sepolia --metrics
 
+#▶️ sepolia.logfile: @ Run an interactive terminal using sepolia network with a log file
+sepolia.logfile: compile-all
+	iex -S mix run -- --checkpoint-sync-url https://sepolia.beaconstate.info --network sepolia --metrics --log-file ./logs/sepolia.log
+
 #▶️ holesky: @ Run an interactive terminal using holesky network
 holesky: compile-all
 	iex -S mix run -- --checkpoint-sync-url https://checkpoint-sync.holesky.ethpandaops.io --network holesky
 
+#▶️ holesky.logfile: @ Run an interactive terminal using holesky network with a log file
+holesky.logfile: compile-all
+	iex -S mix run -- --checkpoint-sync-url https://checkpoint-sync.holesky.ethpandaops.io --network holesky --log-file ./logs/holesky.log
+
 #▶️ gnosis: @ Run an interactive terminal using gnosis network
 gnosis: compile-all
 	iex -S mix run -- --checkpoint-sync-url https://checkpoint.gnosischain.com --network gnosis

config/runtime.exs

@@ -219,7 +219,7 @@ case Keyword.get(args, :log_file) do
   nil ->
     # Use custom formatter for prettier logs
     config :logger, :default_formatter,
-      format: {ConsoleLogger, :format},
+      format: {CustomConsoleLogger, :format},
       metadata: [:slot, :root, :bits]
 
   log_file ->
@@ -241,9 +241,9 @@ case Keyword.get(args, :log_file) do
     config :logger, utc_log: true
 
     config :logger, :default_formatter,
-      format: {LogfmtEx, :format},
+      format: {CustomLogfmtEx, :format},
       colors: [enabled: false],
-      metadata: [:mfa, :slot]
+      metadata: [:mfa, :pid, :slot, :root]
 
     config :logfmt_ex, :opts,
       message_key: "msg",

lib/lambda_ethereum_consensus/logger/console_logger.ex renamed to lib/lambda_ethereum_consensus/logger/custom_console_logger.ex

@@ -1,4 +1,4 @@
-defmodule ConsoleLogger do
+defmodule CustomConsoleLogger do
   @moduledoc """
   Custom logger formatter for console output.
   """

lib/lambda_ethereum_consensus/logger/custom_logfmt_ex.ex

@@ -0,0 +1,19 @@
+defmodule CustomLogfmtEx do
+  @moduledoc """
+  Custom logger formatter for logfmt output.
+  """
+
+  alias LambdaEthereumConsensus.Utils
+
+  def format(level, message, timestamp, metadata) do
+    formatted_metadata = format_metadata(metadata)
+
+    LogfmtEx.format(level, message, timestamp, formatted_metadata)
+  end
+
+  defp format_metadata(metadata) do
+    metadata
+    |> Keyword.replace_lazy(:root, &Utils.format_shorten_binary(&1))
+    |> Keyword.replace_lazy(:bits, &Utils.format_bitstring(&1))
+  end
+end

metrics/loki/loki.yml

@@ -34,7 +34,7 @@ schema_config:
         period: 24h
 
 ruler:
-  alertmanager_url: http://localhost:9093
+  alertmanager_url: http://localhost:9093/alertmanager
 # By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
 # analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
 #

metrics/promtail/promtail.yml

@@ -24,12 +24,18 @@ scrape_configs:
             msg:
             mfa:
             process: registered_name
+            pid:
+            slot:
+            root:
       - timestamp:
           format: "2006-01-02T15:04:05.000"
           source: ts
       - labels:
           level:
           mfa:
           process:
+          pid:
+          slot:
+          root:
       - output:
           source: msg