[tmpnet] Deploy collectors with golang to simplify cross-repo use

Previously, prometheus and promtail were installed and launched by
with bash scripts. Migrating installation to nix and launch to golang
enables directly sharing the functionality with subnet-evm and
hypersdk. No more having to copy and maintain copies of the scripts in
multiple repos.

Author: maru-ava

.github/actions/run-monitored-tmpnet-cmd/action.yml

@@ -41,23 +41,6 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Start prometheus
-      # Only run for the original repo; a forked repo won't have access to the monitoring credentials
-      if: (inputs.prometheus_username != '')
-      shell: bash
-      # Assumes calling project has a nix flake that ensures a compatible prometheus
-      run: nix develop --impure --command bash -x ./scripts/run_prometheus.sh
-      env:
-        PROMETHEUS_USERNAME: ${{ inputs.prometheus_username }}
-        PROMETHEUS_PASSWORD: ${{ inputs.prometheus_password }}
-    - name: Start promtail
-      if: (inputs.prometheus_username != '')
-      shell: bash
-      # Assumes calling project has a nix flake that ensures a compatible promtail
-      run: nix develop --impure --command bash -x ./scripts/run_promtail.sh
-      env:
-        LOKI_USERNAME: ${{ inputs.loki_username }}
-        LOKI_PASSWORD: ${{ inputs.loki_password }}
     - name: Notify of metrics availability
       if: (inputs.prometheus_username != '')
       shell: bash
@@ -68,9 +51,14 @@ runs:
         FILTER_BY_OWNER: ${{ inputs.filter_by_owner }}
     - name: Run command
       shell: bash
-      run: ${{ inputs.run_env }} ${{ inputs.run }}
+      # --impure ensures the env vars are accessible to the command
+      run: ${{ inputs.run_env }} nix develop --impure --command bash -x ${{ inputs.run }}
       env:
-        TMPNET_DELAY_NETWORK_SHUTDOWN: true # Ensure shutdown waits for a final metrics scrape
+        TMPNET_ENABLE_COLLECTORS: true
+        LOKI_USERNAME: ${{ inputs.loki_username }}
+        LOKI_PASSWORD: ${{ inputs.loki_password }}
+        PROMETHEUS_USERNAME: ${{ inputs.prometheus_username }}
+        PROMETHEUS_PASSWORD: ${{ inputs.prometheus_password }}
         GH_REPO: ${{ inputs.repository_owner }}/${{ inputs.repository_name }}
         GH_WORKFLOW: ${{ inputs.workflow }}
         GH_RUN_ID: ${{ inputs.run_id }}

scripts/run_prometheus.sh

@@ -130,6 +130,10 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
 		}
 	}
 
+	if flagVars.EnableCollectors() {
+		require.NoError(tmpnet.EnsureCollectorsRunning(tc.Log()))
+	}
+
 	// Start a new network
 	if network == nil {
 		network = desiredNetwork

scripts/run_promtail.sh

@@ -15,24 +15,16 @@ import (
 	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
 )
 
-const (
-	// Ensure that this value takes into account the scrape_interval
-	// defined in scripts/run_prometheus.sh.
-	networkShutdownDelay = 12 * time.Second
-
-	delayNetworkShutdownEnvName = "TMPNET_DELAY_NETWORK_SHUTDOWN"
-)
-
 type FlagVars struct {
-	avalancheGoExecPath  string
-	pluginDir            string
-	networkDir           string
-	reuseNetwork         bool
-	delayNetworkShutdown bool
-	startNetwork         bool
-	stopNetwork          bool
-	restartNetwork       bool
-	nodeCount            int
+	avalancheGoExecPath string
+	pluginDir           string
+	networkDir          string
+	reuseNetwork        bool
+	enableCollectors    bool
+	startNetwork        bool
+	stopNetwork         bool
+	restartNetwork      bool
+	nodeCount           int
 }
 
 func (v *FlagVars) AvalancheGoExecPath() (string, error) {
@@ -80,10 +72,14 @@ func (v *FlagVars) RestartNetwork() bool {
 	return v.restartNetwork
 }
 
+func (v *FlagVars) EnableCollectors() bool {
+	return v.enableCollectors
+}
+
 func (v *FlagVars) NetworkShutdownDelay() time.Duration {
-	if v.delayNetworkShutdown {
+	if v.enableCollectors {
 		// Only return a non-zero value if the delay is enabled.
-		return networkShutdownDelay
+		return tmpnet.NetworkShutdownDelay
 	}
 	return 0
 }
@@ -147,10 +143,10 @@ func RegisterFlags() *FlagVars {
 		"[optional] restart an existing network previously started with --reuse-network. Useful for ensuring a network is running with the current state of binaries on disk. Ignored if a network is not already running or --stop-network is provided.",
 	)
 	flag.BoolVar(
-		&vars.delayNetworkShutdown,
-		"delay-network-shutdown",
-		cast.ToBool(GetEnvWithDefault(delayNetworkShutdownEnvName, "false")),
-		"[optional] whether to delay network shutdown to allow a final metrics scrape.",
+		&vars.enableCollectors,
+		"enable-collectors",
+		cast.ToBool(GetEnvWithDefault("TMPNET_ENABLE_COLLECTORS", "false")),
+		"[optional] whether to enable collectors of logs and metrics from nodes of the temporary network.",
 	)
 	flag.BoolVar(
 		&vars.startNetwork,

tests/fixture/e2e/env.go

@@ -49,7 +49,7 @@ var _ = ginkgo.AfterEach(func() {
 	// Extend the end time by the shutdown delay (a proxy for the metrics
 	// scrape interval) to maximize the chances of the specified duration
 	// including all metrics relevant to the current spec.
-	endTime := time.Now().Add(networkShutdownDelay).UnixMilli()
+	endTime := time.Now().Add(tmpnet.NetworkShutdownDelay).UnixMilli()
 	metricsLink := tmpnet.MetricsLinkForNetwork(
 		env.GetNetwork().UUID,
 		strconv.FormatInt(startTime, 10),

tests/fixture/e2e/flags.go

@@ -226,7 +226,12 @@ func (p *NodeProcess) getProcess() (*os.Process, error) {
 		return nil, nil
 	}
 
-	proc, err := os.FindProcess(p.pid)
+	return getProcess(p.pid)
+}
+
+// getProcess retrieves the process if it is running.
+func getProcess(pid int) (*os.Process, error) {
+	proc, err := os.FindProcess(pid)
 	if err != nil {
 		return nil, fmt.Errorf("failed to find process: %w", err)
 	}