test(mc): Add integration test for Retina-GKE, refactor test dir (#1301)

# Description

* add integration test for Retina on GKE
* add package utils and refactor test directory

## Related Issue

#1267 

## Checklist

- [x] I have read the [contributing
documentation](https://retina.sh/docs/contributing).
- [x] I signed and signed-off the commits (`git commit -S -s ...`). See
[this
documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification)
on signing commits.
- [x] I have correctly attributed the author(s) of the code.
- [x] I have tested the changes locally.
- [x] I have followed the project's style guidelines.
- [x] I have updated the documentation, if necessary.
- [x] I have added tests, if applicable.

## Screenshots (if applicable) or Testing Completed


![image](https://github.com/user-attachments/assets/d80045e7-8342-47f0-89a4-950f4d3b9a4d)

## Additional Notes

Add any additional notes or context about the pull request here.

---

Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more
information on how to contribute to this project.

test/multicloud/Makefile

@@ -45,7 +45,7 @@ kind-kubeconfig:
 # Once we do this targets will be updated to
 # @cd test && go test -v -count=1 -timeout 30m ./...
 test:
-	@cd test && go test -run TestRetinaKindIntegration -count=1 -timeout 10m
+	@cd test/integration && go test -run TestRetinaKindIntegration -count=1 -timeout 20m
 
 fmt:
 	@tofu fmt -recursive

test/multicloud/README.md

@@ -96,3 +96,18 @@ make test
 * [GKE resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster)
 * [AKS resource documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster)
 * [Kind resource documentation](https://registry.terraform.io/providers/tehcyx/kind/latest/docs/resources/cluster)
+
+## Troubleshooting
+
+In case the test fails due to timeout, validate the resource was created by the provider, and if it is, you can import into OpenTofu state.
+
+Here is an example on how to import resources for `modules/gke`
+
+```sh
+# move to the stack directory
+# i.e. examples/gke
+tofu import module.gke.google_container_cluster.gke europe-west2/test-gke-cluster
+tofu import module.gke.google_service_account.default projects/mc-retina/serviceAccounts/[email protected]
+```
+
+>Note: each resource documentation contains a section on how to import resources into the State. [Example for google_container_cluster resource](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#import)

test/multicloud/examples/integration/retina-gke/main.tf

@@ -0,0 +1,14 @@
+module "gke" {
+  source       = "../../../modules/gke"
+  location     = var.location
+  prefix       = var.prefix
+  project      = var.project
+  machine_type = var.machine_type
+}
+
+module "retina" {
+  depends_on = [module.gke]
+  source     = "../../../modules/retina"
+  retina_version = var.retina_version
+  values = var.values
+}

test/multicloud/examples/integration/retina-gke/outputs.tf

@@ -0,0 +1,14 @@
+output "host" {
+  value     = module.gke.host
+  sensitive = true
+}
+
+output "cluster_ca_certificate" {
+  value     = module.gke.cluster_ca_certificate
+  sensitive = true
+}
+
+output "access_token" {
+  value     = data.google_client_config.current.access_token
+  sensitive = true
+}

test/multicloud/examples/integration/retina-gke/providers.tf

@@ -0,0 +1,30 @@
+terraform {
+  required_version = "1.8.3"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.17.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.17.0"
+    }
+  }
+}
+
+# Initialize the Google provider
+provider "google" {
+  project = var.project
+  region  = var.location
+}
+
+data "google_client_config" "current" {}
+
+# Initialize the Helm provider
+provider "helm" {
+  kubernetes {
+    token                  = data.google_client_config.current.access_token
+    host                   = module.gke.host
+    cluster_ca_certificate = base64decode(module.gke.cluster_ca_certificate)
+  }
+}

test/multicloud/examples/integration/retina-gke/variables.tf

@@ -0,0 +1,36 @@
+variable "project" {
+  description = "The Google Cloud project where resources will be deployed."
+  type        = string
+  default     = "mc-retina"
+}
+
+variable "location" {
+  description = "The Google Cloud location where GKE will be deployed to."
+  type        = string
+  default     = "eu-west2"
+}
+
+variable "prefix" {
+  description = "A prefix to add to all resources."
+  type        = string
+  default     = "mc"
+}
+
+variable "machine_type" {
+  description = "The machine type to use for the GKE nodes."
+  type        = string
+  default     = "e2-standard-4"
+}
+
+variable "retina_version" {
+  description = "The tag to apply to all resources."
+  type        = string
+}
+
+variable "values" {
+  description = "Configuration for set blocks, this corresponds to Helm values.yaml"
+  type = list(object({
+    name  = string
+    value = string
+  }))
+}

test/multicloud/modules/gke/main.tf

@@ -3,6 +3,7 @@ resource "google_service_account" "default" {
   display_name = "GKE Service Account for ${var.project}"
 }
 
+// https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview
 resource "google_container_cluster" "gke" {
   name     = "${var.prefix}-gke-cluster"
   location = var.location

test/multicloud/test/integration_prometheus_kind_test.go → test/multicloud/test/integration/prometheus_kind_test.go

@@ -5,13 +5,14 @@ import (
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/retina/test/multicloud/test/utils"
 )
 
 func TestPrometheusKindIntegration(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: examplesPath + "integration/prometheus-kind",
+		TerraformDir: utils.ExamplesPath + "integration/prometheus-kind",
 
 		Vars: map[string]interface{}{
 			"prefix": "test-integration",
@@ -23,38 +24,38 @@ func TestPrometheusKindIntegration(t *testing.T) {
 	terraform.InitAndApply(t, opts)
 
 	// get outputs
-	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
-	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
-	clientKey := fetchSensitiveOutput(t, opts, "client_key")
-	host := fetchSensitiveOutput(t, opts, "host")
+	caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := utils.FetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := utils.FetchSensitiveOutput(t, opts, "client_key")
+	host := utils.FetchSensitiveOutput(t, opts, "host")
 
 	// build the REST config
-	restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
+	restConfig := utils.CreateRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
 
 	// create a Kubernetes clientset
-	clientSet, err := buildClientSet(restConfig)
+	clientSet, err := utils.BuildClientSet(restConfig)
 	if err != nil {
 		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
 	}
 
 	// test the cluster is accessible
-	testClusterAccess(t, clientSet)
+	utils.TestClusterAccess(t, clientSet)
 
-	podSelector := PodSelector{
+	podSelector := utils.PodSelector{
 		Namespace:     "default",
 		LabelSelector: "app.kubernetes.io/instance=prometheus-kube-prometheus-prometheus",
 		ContainerName: "prometheus",
 	}
 
 	timeOut := time.Duration(60) * time.Second
 	// check the prometheus pods are running
-	result, err := arePodsRunning(clientSet, podSelector, timeOut)
+	result, err := utils.ArePodsRunning(clientSet, podSelector, timeOut)
 	if !result {
 		t.Fatalf("Prometheus pods did not start in time: %v\n", err)
 	}
 
 	// check the retina pods logs for errors
-	checkPodLogs(t, clientSet, podSelector)
+	utils.CheckPodLogs(t, clientSet, podSelector)
 
 	// TODO: add more tests here
 }

test/multicloud/test/integration/retina_gke_test.go

@@ -0,0 +1,81 @@
+package test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/retina/test/multicloud/test/utils"
+)
+
+func TestRetinaGKEIntegration(t *testing.T) {
+	t.Parallel()
+
+	opts := &terraform.Options{
+		TerraformDir: utils.ExamplesPath + "integration/retina-gke",
+
+		Vars: map[string]interface{}{
+			"prefix":         "test",
+			"location":       "europe-west2", // London
+			"project":        "mc-retina",    // TODO: replace with actual project once we get gcloud access
+			"machine_type":   "e2-standard-4",
+			"retina_version": utils.RetinaVersion,
+			"values": []map[string]interface{}{
+				{
+					"name":  "logLevel",
+					"value": "info",
+				},
+				{
+					"name":  "operator.tag",
+					"value": utils.RetinaVersion,
+				},
+				{
+					"name":  "image.tag",
+					"value": utils.RetinaVersion,
+				},
+			},
+		},
+	}
+
+	// clean up at the end of the test
+	defer terraform.Destroy(t, opts)
+	terraform.InitAndApply(t, opts)
+
+	// get outputs
+	caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	host := utils.FetchSensitiveOutput(t, opts, "host")
+	token := utils.FetchSensitiveOutput(t, opts, "access_token")
+
+	// decode the base64 encoded cert
+	caCertString := utils.DecodeBase64(t, caCert)
+
+	// build the REST config
+	restConfig := utils.CreateRESTConfigWithBearer(caCertString, token, host)
+
+	// create a Kubernetes clientset
+	clientSet, err := utils.BuildClientSet(restConfig)
+	if err != nil {
+		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
+	}
+
+	// test the cluster is accessible
+	utils.TestClusterAccess(t, clientSet)
+
+	retinaPodSelector := utils.PodSelector{
+		Namespace:     "kube-system",
+		LabelSelector: "k8s-app=retina",
+		ContainerName: "retina",
+	}
+
+	timeOut := time.Duration(90) * time.Second
+	// check the retina pods are running
+	result, err := utils.ArePodsRunning(clientSet, retinaPodSelector, timeOut)
+	if !result {
+		t.Fatalf("Retina pods did not start in time: %v\n", err)
+	}
+
+	// check the retina pods logs for errors
+	utils.CheckPodLogs(t, clientSet, retinaPodSelector)
+
+	// TODO: add more tests here
+}