K8SPG-719 PMM3 support (#1084)

* K8SPG-719

* fix reviewdog

* e2e test for pmm3

* run shfmt and introduce make command

* handle nil secret & add unit test case

* cr: remove pmm server helm version in order to always use the latest always

* Revert "run shfmt and introduce make command"

This reverts commit 897e1c8.

* changes on functions after revert

* add empty lines

* cr: specify in the log PMM3 is used with token and PMM2 with api key

---------

Co-authored-by: Viacheslav Sarzhan <[email protected]>

Author: gkech

deploy/secrets.yaml

@@ -5,6 +5,7 @@ metadata:
 type: Opaque
 stringData:
   PMM_SERVER_KEY: ""
+  #PMM_SERVER_TOKEN: ""
 ---
 apiVersion: v1
 kind: Secret

e2e-tests/functions

@@ -365,6 +365,41 @@ deploy_pmm_server() {
 	fi
 }
 
+deploy_pmm3_server() {
+	helm uninstall -n "${NAMESPACE}" pmm || :
+	if [[ $OPENSHIFT ]]; then
+		platform=openshift
+		oc create sa pmm-server -n "$NAMESPACE"
+		oc adm policy add-scc-to-user privileged -z pmm-server -n "$NAMESPACE"
+
+		if [[ $OPERATOR_NS ]]; then
+			timeout 30 oc delete clusterrolebinding $(kubectl get clusterrolebinding | grep 'pmm-pg-operator-' | awk '{print $1}') || :
+			oc create clusterrolebinding pmm-pg-operator-cluster-wide --clusterrole=percona-postgresql-operator --serviceaccount=$NAMESPACE:pmm-server -n "$NAMESPACE"
+			oc patch clusterrole/percona-postgresql-operator --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' ${OPERATOR_NS:+-n $OPERATOR_NS}
+		else
+			oc create rolebinding pmm-pg-operator-namespace-only --role percona-postgresql-operator --serviceaccount=$NAMESPACE:pmm-server -n "${NAMESPACE}"
+			oc patch role/percona-postgresql-operator --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' -n "$NAMESPACE"
+		fi
+		helm install monitoring --set imageTag=${IMAGE_PMM3_SERVER#*:} --set imageRepo=${IMAGE_PMM3_SERVER%:*} --set platform=$platform --set sa=pmm-server --set supresshttp2=false https://percona-charts.storage.googleapis.com/pmm-server-${PMM3_SERVER_VERSION}.tgz -n "$NAMESPACE"
+	else
+		platform=kubernetes
+
+		helm uninstall -n "${NAMESPACE}" monitoring || :
+		helm repo remove percona || :
+		kubectl delete clusterrole monitoring --ignore-not-found
+		kubectl delete clusterrolebinding monitoring --ignore-not-found
+
+		helm repo add percona https://percona.github.io/percona-helm-charts/
+		helm install monitoring percona/pmm -n "${NAMESPACE}" \
+			--set fullnameOverride=monitoring \
+			--set imageTag=3-dev-latest \
+			--set imageRepo=perconalab/pmm-server \
+			--set service.type=LoadBalancer \
+			--set platform="$platform" \
+			--force
+	fi
+}
+
 generate_pmm_api_key() {
 	local ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2")
 	local PMM_SERVICE_IP=$(get_service_ip monitoring-service)
@@ -378,17 +413,71 @@ generate_pmm_api_key() {
 		| jq -r .key
 }
 
+generate_pmm3_server_token() {
+	local key_name=$RANDOM
+
+	local ADMIN_PASSWORD
+	ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" get secret pmm-secret -o jsonpath="{.data.PMM_ADMIN_PASSWORD}" | base64 --decode)
+
+	if [[ -z $ADMIN_PASSWORD ]]; then
+		echo "Error: ADMIN_PASSWORD is empty or not found!" >&2
+		return 1
+	fi
+
+	local create_response create_status_code create_json_response
+	create_response=$(curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' \
+		-d "{\"name\":\"${key_name}\", \"role\":\"Admin\", \"isDisabled\":false}" \
+		--user "admin:${ADMIN_PASSWORD}" \
+		"https://$(get_service_ip monitoring-service)/graph/api/serviceaccounts" \
+		-w "\n%{http_code}")
+
+	create_status_code=$(echo "$create_response" | tail -n1)
+	create_json_response=$(echo "$create_response" | sed '$ d')
+
+	if [[ $create_status_code -ne 201 ]]; then
+		echo "Error: Failed to create PMM service account. HTTP Status: $create_status_code" >&2
+		echo "Response: $create_json_response" >&2
+		return 1
+	fi
+
+	local service_account_id
+	service_account_id=$(echo "$create_json_response" | jq -r '.id')
+
+	if [[ -z $service_account_id || $service_account_id == "null" ]]; then
+		echo "Error: Failed to extract service account ID!" >&2
+		return 1
+	fi
+
+	local token_response token_status_code token_json_response
+	token_response=$(curl --insecure -s -X POST -H 'Content-Type: application/json' \
+		-d "{\"name\":\"${key_name}\"}" \
+		--user "admin:${ADMIN_PASSWORD}" \
+		"https://$(get_service_ip monitoring-service)/graph/api/serviceaccounts/${service_account_id}/tokens" \
+		-w "\n%{http_code}")
+
+	token_status_code=$(echo "$token_response" | tail -n1)
+	token_json_response=$(echo "$token_response" | sed '$ d')
+
+	if [[ $token_status_code -ne 200 ]]; then
+		echo "Error: Failed to create token. HTTP Status: $token_status_code" >&2
+		echo "Response: $token_json_response" >&2
+		return 1
+	fi
+
+	echo "$token_json_response" | jq -r '.key'
+}
+
 get_metric_values() {
 	local metric=$1
 	local instance=$2
-	local api_key=$3
+	local token=$3
 	local start=$($date -u "+%s" -d "-5 minute")
 	local end=$($date -u "+%s")
 	local endpoint=$(get_service_ip monitoring-service)
 
 	local wait_count=20
 	local retry=0
-	until [[ $(curl -s -k -H "Authorization: Bearer ${api_key}" "https://$endpoint/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28$metric%7Bnode_name%3D%7E%22$instance%22%7d%20or%20$metric%7Bnode_name%3D%7E%22$instance%22%7D%29&start=$start&end=$end&step=60" \
+	until [[ $(curl -s -k -H "Authorization: Bearer ${token}" "https://$endpoint/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28$metric%7Bnode_name%3D%7E%22$instance%22%7d%20or%20$metric%7Bnode_name%3D%7E%22$instance%22%7D%29&start=$start&end=$end&step=60" \
 		| jq '.data.result[0].values[][1]' \
 		| grep '^"[0-9]') ]]; do
 		sleep 2
@@ -438,6 +527,43 @@ EOF
 	rm -f payload.json
 }
 
+get_qan20_values_pmm3() {
+	local instance=$1
+	local token=$2
+	local start=$($date -u "+%Y-%m-%dT%H:%M:%S" -d "-30 minute")
+	local end=$($date -u "+%Y-%m-%dT%H:%M:%S")
+	local endpoint=$(get_service_ip monitoring-service)
+
+	cat >payload.json <<EOF
+{
+   "columns":[
+	  "load",
+	  "num_queries",
+	  "query_time"
+   ],
+   "first_seen": false,
+   "group_by": "queryid",
+   "include_only_fields": [],
+   "keyword": "",
+   "labels": [
+	   {
+		   "key": "cluster",
+		   "value": ["postgresql"]
+   }],
+   "limit": 10,
+   "offset": 0,
+   "order_by": "-load",
+   "main_metric": "load",
+   "period_start_from": "$($date -u -d '-12 hour' '+%Y-%m-%dT%H:%M:%S%:z')",
+   "period_start_to": "$($date -u '+%Y-%m-%dT%H:%M:%S%:z')"
+}
+EOF
+
+	curl -s -k -H "Authorization: Bearer ${token}" -XPOST -d @payload.json "https://$endpoint/v1/qan/metrics:getReport" \
+		| jq '.rows[].sparkline'
+	rm -f payload.json
+}
+
 deploy_chaos_mesh() {
 	destroy_chaos_mesh
 

e2e-tests/run-pr.csv

@@ -4,6 +4,7 @@ demand-backup
 finalizers
 init-deploy
 monitoring
+monitoring-pmm3
 one-pod
 operator-self-healing
 pitr

e2e-tests/run-release.csv

@@ -6,6 +6,7 @@ init-deploy
 major-upgrade
 migration-backup-s3
 monitoring
+monitoring-pmm3
 one-pod
 operator-self-healing
 pitr

e2e-tests/tests/monitoring-pmm3/00-assert.yaml

@@ -0,0 +1,24 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: perconapgclusters.pgv2.percona.com
+spec:
+  group: pgv2.percona.com
+  names:
+    kind: PerconaPGCluster
+    listKind: PerconaPGClusterList
+    plural: perconapgclusters
+    singular: perconapgcluster
+  scope: Namespaced
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-operator-deploy-status
+timeout: 120
+commands:
+  - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1

e2e-tests/tests/monitoring-pmm3/00-deploy-operator.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+      init_temp_dir # do this only in the first TestStep
+
+      deploy_operator
+      deploy_client

e2e-tests/tests/monitoring-pmm3/01-assert.yaml

@@ -0,0 +1,33 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 520
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: monitoring
+status:
+  collisionCount: 0
+  currentReplicas: 1
+  observedGeneration: 1
+  readyReplicas: 1
+  replicas: 1
+  updatedReplicas: 1
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: monitoring-service
+---
+apiVersion: v1
+count: 1
+involvedObject:
+  apiVersion: v1
+  kind: Service
+  name: monitoring-service
+kind: Event
+message: Ensured load balancer
+reason: EnsuredLoadBalancer
+source:
+  component: service-controller
+type: Normal

e2e-tests/tests/monitoring-pmm3/01-deploy-pmm-server.yaml

@@ -0,0 +1,11 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      deploy_pmm3_server
+    timeout: 300

e2e-tests/tests/monitoring-pmm3/02-assert.yaml

@@ -0,0 +1,8 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: monitoring-pmm3-pmm-secret

e2e-tests/tests/monitoring-pmm3/02-create-pmm-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+      kubectl create -n "${NAMESPACE}" secret generic monitoring-pmm3-pmm-secret --from-literal=PMM_SERVER_TOKEN="" || true

e2e-tests/tests/monitoring-pmm3/03-assert.yaml

@@ -0,0 +1,99 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 420
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  labels:
+    e2e: monitoring-pmm3
+  name: monitoring-pmm3
+spec:
+  backups:
+    pgbackrest:
+      manual:
+        options:
+          - --type=full
+        repoName: repo1
+      repos:
+        - name: repo1
+          schedules:
+            full: 0 0 * * 6
+          volume:
+            volumeClaimSpec:
+              accessModes:
+                - ReadWriteOnce
+              resources:
+                requests:
+                  storage: 1Gi
+  instances:
+    - dataVolumeClaimSpec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+      name: instance1
+      replicas: 3
+  pmm:
+    enabled: true
+    secret: monitoring-pmm3-pmm-secret
+    serverHost: monitoring-service
+  port: 5432
+  proxy:
+    pgBouncer:
+      port: 5432
+      replicas: 3
+status:
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+      - name: instance1
+        ready: 3
+        size: 3
+    ready: 3
+    size: 3
+  state: ready
+---
+kind: Job
+apiVersion: batch/v1
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: monitoring-pmm3
+    postgres-operator.crunchydata.com/pgbackrest: ''
+    postgres-operator.crunchydata.com/pgbackrest-backup: replica-create
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGBackup
+      controller: true
+      blockOwnerDeletion: true
+status:
+  succeeded: 1
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  annotations:
+    pgv2.percona.com/pgbackrest-backup-job-type: replica-create
+  generation: 1
+spec:
+  pgCluster: monitoring-pmm3
+  repoName: repo1
+status:
+  backupType: full
+  repo:
+    name: repo1
+    schedules:
+      full: 0 0 * * 6
+    volume:
+      volumeClaimSpec:
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+  state: Succeeded
+  storageType: filesystem