Skip to content

Commit a1beefd

Browse files
SRodiSRodi
authored
test(mc): Add integration test for Retina-GKE, refactor test dir (#1301)
# Description * add integration test for Retina on GKE * add package utils and refactor test directory ## Related Issue #1267 ## Checklist - [x] I have read the [contributing documentation](https://retina.sh/docs/contributing). - [x] I signed and signed-off the commits (`git commit -S -s ...`). See [this documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) on signing commits. - [x] I have correctly attributed the author(s) of the code. - [x] I have tested the changes locally. - [x] I have followed the project's style guidelines. - [x] I have updated the documentation, if necessary. - [x] I have added tests, if applicable. ## Screenshots (if applicable) or Testing Completed ![image](https://github.com/user-attachments/assets/d80045e7-8342-47f0-89a4-950f4d3b9a4d) ## Additional Notes Add any additional notes or context about the pull request here. --- Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more information on how to contribute to this project.
1 parent 04fe95a commit a1beefd

18 files changed

+342
-105
lines changed

‎test/multicloud/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ kind-kubeconfig:
4545
# Once we do this targets will be updated to
4646
# @cd test && go test -v -count=1 -timeout 30m ./...
4747
test:
48-
@cd test && go test -run TestRetinaKindIntegration -count=1 -timeout 10m
48+
@cd test/integration && go test -run TestRetinaKindIntegration -count=1 -timeout 20m
4949

5050
fmt:
5151
@tofu fmt -recursive

‎test/multicloud/README.md

+15
Original file line numberDiff line numberDiff line change
@@ -96,3 +96,18 @@ make test
9696
* [GKE resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster)
9797
* [AKS resource documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster)
9898
* [Kind resource documentation](https://registry.terraform.io/providers/tehcyx/kind/latest/docs/resources/cluster)
99+
100+
## Troubleshooting
101+
102+
In case the test fails due to timeout, validate the resource was created by the provider, and if it is, you can import into OpenTofu state.
103+
104+
Here is an example on how to import resources for `modules/gke`
105+
106+
```sh
107+
# move to the stack directory
108+
# i.e. examples/gke
109+
tofu import module.gke.google_container_cluster.gke europe-west2/test-gke-cluster
110+
tofu import module.gke.google_service_account.default projects/mc-retina/serviceAccounts/[email protected]
111+
```
112+
113+
>Note: each resource documentation contains a section on how to import resources into the State. [Example for google_container_cluster resource](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#import)

‎test/multicloud/examples/integration/retina-gke/.terraform.lock.hcl

+38
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/multicloud/examples/integration/retina-gke/main.tf

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
module "gke" {
2+
source = "../../../modules/gke"
3+
location = var.location
4+
prefix = var.prefix
5+
project = var.project
6+
machine_type = var.machine_type
7+
}
8+
9+
module "retina" {
10+
depends_on = [module.gke]
11+
source = "../../../modules/retina"
12+
retina_version = var.retina_version
13+
values = var.values
14+
}

‎test/multicloud/examples/integration/retina-gke/outputs.tf

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
output "host" {
2+
value = module.gke.host
3+
sensitive = true
4+
}
5+
6+
output "cluster_ca_certificate" {
7+
value = module.gke.cluster_ca_certificate
8+
sensitive = true
9+
}
10+
11+
output "access_token" {
12+
value = data.google_client_config.current.access_token
13+
sensitive = true
14+
}

‎test/multicloud/examples/integration/retina-gke/providers.tf

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
terraform {
2+
required_version = "1.8.3"
3+
required_providers {
4+
google = {
5+
source = "hashicorp/google"
6+
version = "6.17.0"
7+
}
8+
helm = {
9+
source = "hashicorp/helm"
10+
version = "2.17.0"
11+
}
12+
}
13+
}
14+
15+
# Initialize the Google provider
16+
provider "google" {
17+
project = var.project
18+
region = var.location
19+
}
20+
21+
data "google_client_config" "current" {}
22+
23+
# Initialize the Helm provider
24+
provider "helm" {
25+
kubernetes {
26+
token = data.google_client_config.current.access_token
27+
host = module.gke.host
28+
cluster_ca_certificate = base64decode(module.gke.cluster_ca_certificate)
29+
}
30+
}

‎test/multicloud/examples/integration/retina-gke/variables.tf

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
variable "project" {
2+
description = "The Google Cloud project where resources will be deployed."
3+
type = string
4+
default = "mc-retina"
5+
}
6+
7+
variable "location" {
8+
description = "The Google Cloud location where GKE will be deployed to."
9+
type = string
10+
default = "eu-west2"
11+
}
12+
13+
variable "prefix" {
14+
description = "A prefix to add to all resources."
15+
type = string
16+
default = "mc"
17+
}
18+
19+
variable "machine_type" {
20+
description = "The machine type to use for the GKE nodes."
21+
type = string
22+
default = "e2-standard-4"
23+
}
24+
25+
variable "retina_version" {
26+
description = "The tag to apply to all resources."
27+
type = string
28+
}
29+
30+
variable "values" {
31+
description = "Configuration for set blocks, this corresponds to Helm values.yaml"
32+
type = list(object({
33+
name = string
34+
value = string
35+
}))
36+
}

‎test/multicloud/modules/gke/main.tf

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ resource "google_service_account" "default" {
33
display_name = "GKE Service Account for ${var.project}"
44
}
55

6+
// https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview
67
resource "google_container_cluster" "gke" {
78
name = "${var.prefix}-gke-cluster"
89
location = var.location

‎test/multicloud/test/example_kind_test.go

-41
This file was deleted.

‎test/multicloud/test/integration_prometheus_kind_test.go renamed to ‎test/multicloud/test/integration/prometheus_kind_test.go

+12-11
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@ import (
55
"time"
66

77
"github.com/gruntwork-io/terratest/modules/terraform"
8+
"github.com/microsoft/retina/test/multicloud/test/utils"
89
)
910

1011
func TestPrometheusKindIntegration(t *testing.T) {
1112
t.Parallel()
1213

1314
opts := &terraform.Options{
14-
TerraformDir: examplesPath + "integration/prometheus-kind",
15+
TerraformDir: utils.ExamplesPath + "integration/prometheus-kind",
1516

1617
Vars: map[string]interface{}{
1718
"prefix": "test-integration",
@@ -23,38 +24,38 @@ func TestPrometheusKindIntegration(t *testing.T) {
2324
terraform.InitAndApply(t, opts)
2425

2526
// get outputs
26-
caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
27-
clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
28-
clientKey := fetchSensitiveOutput(t, opts, "client_key")
29-
host := fetchSensitiveOutput(t, opts, "host")
27+
caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
28+
clientCert := utils.FetchSensitiveOutput(t, opts, "client_certificate")
29+
clientKey := utils.FetchSensitiveOutput(t, opts, "client_key")
30+
host := utils.FetchSensitiveOutput(t, opts, "host")
3031

3132
// build the REST config
32-
restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
33+
restConfig := utils.CreateRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
3334

3435
// create a Kubernetes clientset
35-
clientSet, err := buildClientSet(restConfig)
36+
clientSet, err := utils.BuildClientSet(restConfig)
3637
if err != nil {
3738
t.Fatalf("Failed to create Kubernetes clientset: %v", err)
3839
}
3940

4041
// test the cluster is accessible
41-
testClusterAccess(t, clientSet)
42+
utils.TestClusterAccess(t, clientSet)
4243

43-
podSelector := PodSelector{
44+
podSelector := utils.PodSelector{
4445
Namespace: "default",
4546
LabelSelector: "app.kubernetes.io/instance=prometheus-kube-prometheus-prometheus",
4647
ContainerName: "prometheus",
4748
}
4849

4950
timeOut := time.Duration(60) * time.Second
5051
// check the prometheus pods are running
51-
result, err := arePodsRunning(clientSet, podSelector, timeOut)
52+
result, err := utils.ArePodsRunning(clientSet, podSelector, timeOut)
5253
if !result {
5354
t.Fatalf("Prometheus pods did not start in time: %v\n", err)
5455
}
5556

5657
// check the retina pods logs for errors
57-
checkPodLogs(t, clientSet, podSelector)
58+
utils.CheckPodLogs(t, clientSet, podSelector)
5859

5960
// TODO: add more tests here
6061
}

‎test/multicloud/test/integration/retina_gke_test.go

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
package test
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/gruntwork-io/terratest/modules/terraform"
8+
"github.com/microsoft/retina/test/multicloud/test/utils"
9+
)
10+
11+
func TestRetinaGKEIntegration(t *testing.T) {
12+
t.Parallel()
13+
14+
opts := &terraform.Options{
15+
TerraformDir: utils.ExamplesPath + "integration/retina-gke",
16+
17+
Vars: map[string]interface{}{
18+
"prefix": "test",
19+
"location": "europe-west2", // London
20+
"project": "mc-retina", // TODO: replace with actual project once we get gcloud access
21+
"machine_type": "e2-standard-4",
22+
"retina_version": utils.RetinaVersion,
23+
"values": []map[string]interface{}{
24+
{
25+
"name": "logLevel",
26+
"value": "info",
27+
},
28+
{
29+
"name": "operator.tag",
30+
"value": utils.RetinaVersion,
31+
},
32+
{
33+
"name": "image.tag",
34+
"value": utils.RetinaVersion,
35+
},
36+
},
37+
},
38+
}
39+
40+
// clean up at the end of the test
41+
defer terraform.Destroy(t, opts)
42+
terraform.InitAndApply(t, opts)
43+
44+
// get outputs
45+
caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
46+
host := utils.FetchSensitiveOutput(t, opts, "host")
47+
token := utils.FetchSensitiveOutput(t, opts, "access_token")
48+
49+
// decode the base64 encoded cert
50+
caCertString := utils.DecodeBase64(t, caCert)
51+
52+
// build the REST config
53+
restConfig := utils.CreateRESTConfigWithBearer(caCertString, token, host)
54+
55+
// create a Kubernetes clientset
56+
clientSet, err := utils.BuildClientSet(restConfig)
57+
if err != nil {
58+
t.Fatalf("Failed to create Kubernetes clientset: %v", err)
59+
}
60+
61+
// test the cluster is accessible
62+
utils.TestClusterAccess(t, clientSet)
63+
64+
retinaPodSelector := utils.PodSelector{
65+
Namespace: "kube-system",
66+
LabelSelector: "k8s-app=retina",
67+
ContainerName: "retina",
68+
}
69+
70+
timeOut := time.Duration(90) * time.Second
71+
// check the retina pods are running
72+
result, err := utils.ArePodsRunning(clientSet, retinaPodSelector, timeOut)
73+
if !result {
74+
t.Fatalf("Retina pods did not start in time: %v\n", err)
75+
}
76+
77+
// check the retina pods logs for errors
78+
utils.CheckPodLogs(t, clientSet, retinaPodSelector)
79+
80+
// TODO: add more tests here
81+
}

0 commit comments

Comments
 (0)