K8SPG-708 replace ready/live probe http check with custom command, change pg entrypoint (#1099)

Author: gkech

build/crd/crunchy/generated/postgres-operator.crunchydata.com_postgresclusters.yaml

@@ -63,6 +63,9 @@ COPY --from=go_builder /usr/local/bin/pgbackrest /usr/local/bin/
 COPY --from=go_builder /licenses /licenses
 COPY build/postgres-operator/install-extensions.sh /usr/local/bin
 COPY build/postgres-operator/init-entrypoint.sh /usr/local/bin
+COPY build/postgres-operator/postgres-entrypoint.sh /usr/local/bin
+COPY build/postgres-operator/postgres-liveness-check.sh /usr/local/bin
+COPY build/postgres-operator/postgres-readiness-check.sh /usr/local/bin
 COPY hack/tools/queries /opt/crunchy/conf
 
 RUN chgrp -R 0 /opt/crunchy/conf && chmod -R g=u opt/crunchy/conf

build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml

@@ -6,3 +6,6 @@ set -o xtrace
 CRUNCHY_BINDIR="/opt/crunchy"
 
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/pgbackrest" "${CRUNCHY_BINDIR}/bin/pgbackrest"
+install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/postgres-entrypoint.sh" "${CRUNCHY_BINDIR}/bin/postgres-entrypoint.sh"
+install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/postgres-liveness-check.sh" "${CRUNCHY_BINDIR}/bin/postgres-liveness-check.sh"
+install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/postgres-readiness-check.sh" "${CRUNCHY_BINDIR}/bin/postgres-readiness-check.sh"

build/postgres-operator/Dockerfile

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# /pgdata/ is already mounted to the pg database container with rw permissions
+recovery_file='/pgdata/sleep-forever'
+if [[ -f ${recovery_file} ]]; then
+	set +o xtrace
+	echo "The $recovery_file file is detected, node entered an infinite sleep"
+	echo "If you want to exit from the infinite sleep, remove the $recovery_file file"
+
+	if [[ ! -d /tmp/postgres ]]; then
+		mkdir -p /tmp/postgres
+	fi
+
+	while [ -f "${recovery_file}" ]; do
+		sleep 3
+	done
+	exit 0
+fi
+
+exec "$@"

build/postgres-operator/init-entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+PATRONI_PORT=8008
+PATRONI_HOST=localhost
+
+# /pgdata/ is already mounted to the pg database container with rw permissions
+recovery_file='/pgdata/sleep-forever'
+if [ -f "${recovery_file}" ]; then
+	set +o xtrace
+	echo "The $recovery_file file is detected, node entered an infinite sleep"
+	echo "If you want to exit from the infinite sleep, remove the $recovery_file file"
+	exit 0
+fi
+
+response=$(curl -s -o /dev/null -w "%{http_code}" -k "https://${PATRONI_HOST}:${PATRONI_PORT}/liveness")
+
+if [[ $response -eq 200 ]]; then
+	exit 0
+fi
+exit 1

build/postgres-operator/postgres-entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+PATRONI_PORT=8008
+PATRONI_HOST=localhost
+
+response=$(curl -s -o /dev/null -w "%{http_code}" -k "https://${PATRONI_HOST}:${PATRONI_PORT}/readiness")
+
+if [[ $response -eq 200 ]]; then
+	exit 0
+fi
+exit 1

build/postgres-operator/postgres-liveness-check.sh

@@ -174,6 +174,31 @@ spec:
   instances:
   - name: instance1
     replicas: 3
+#    initContainer:
+#      image: perconalab/percona-postgresql-operator:main
+#      resources:
+#        limits:
+#          cpu: 2.0
+#          memory: 4Gi
+#      containerSecurityContext:
+#        fsGroup: 1001
+#        runAsUser: 1001
+#        runAsNonRoot: true
+#        fsGroupChangePolicy: "OnRootMismatch"
+#        runAsGroup: 1001
+#        seLinuxOptions:
+#          type: spc_t
+#          level: s0:c123,c456
+#        seccompProfile:
+#          type: Localhost
+#          localhostProfile: localhost/profile.json
+#        supplementalGroups:
+#        - 1001
+#        sysctls:
+#        - name: net.ipv4.tcp_keepalive_time
+#          value: "600"
+#        - name: net.ipv4.tcp_keepalive_intvl
+#          value: "60"
 
     affinity:
       podAntiAffinity:

build/postgres-operator/postgres-readiness-check.sh

@@ -178,6 +178,7 @@ get_cr() {
 		.spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] |
 		.spec.users += [{"name":"'${cr_name}'","password":{"type":"AlphaNumeric"}}] |
 		.spec.image = "'$IMAGE_POSTGRESQL'" |
+		.spec.initContainer.image = "perconalab/percona-postgresql-operator:K8SPG-708-12" |
 		.spec.backups.pgbackrest.image = "'$IMAGE_BACKREST'" |
 		.spec.proxy.pgBouncer.image = "'$IMAGE_PGBOUNCER'" |
 		.spec.pmm.image = "'$IMAGE_PMM_CLIENT'" |

config/crd/bases/pgv2.percona.com_perconapgclusters.yaml

@@ -52,6 +52,8 @@ spec:
         - name: postgres-data
         - mountPath: /tmp
           name: tmp
+      - command:
+        - /usr/local/bin/init-entrypoint.sh
       - name: nss-wrapper-init
 status:
   observedGeneration: 2

config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml

@@ -52,6 +52,8 @@ spec:
         - name: postgres-data
         - mountPath: /tmp
           name: tmp
+      - command:
+        - /usr/local/bin/init-entrypoint.sh
       - name: nss-wrapper-init
 status:
   observedGeneration: 2

deploy/bundle.yaml

@@ -54,6 +54,8 @@ spec:
         - name: postgres-data
         - mountPath: /tmp
           name: tmp
+      - command:
+        - /usr/local/bin/init-entrypoint.sh
       - name: nss-wrapper-init
 status:
   observedGeneration: 3

deploy/cr.yaml

@@ -24,6 +24,8 @@ spec:
         - /usr/local/bin/relocate-extensions.sh
       - command:
         - /usr/local/bin/install-extensions.sh
+      - command:
+        - /usr/local/bin/init-entrypoint.sh
       - name: nss-wrapper-init
 status:
   observedGeneration: 3
@@ -47,4 +49,4 @@ status:
       size: 3
     ready: 3
     size: 3
-  state: ready
+  state: ready

deploy/crd.yaml

@@ -144,11 +144,17 @@ func TestCustomLabels(t *testing.T) {
 		cluster.ObjectMeta.Name = "global-cluster"
 		cluster.ObjectMeta.Namespace = ns.Name
 		cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{
-			Name:                "daisy-instance1",
+			Name: "daisy-instance1",
+			InitContainer: &v1beta1.InitContainerSpec{
+				Image: "some-image",
+			},
 			Replicas:            initialize.Int32(1),
 			DataVolumeClaimSpec: testVolumeClaimSpec(),
 		}, {
-			Name:                "daisy-instance2",
+			Name: "daisy-instance2",
+			InitContainer: &v1beta1.InitContainerSpec{
+				Image: "some-image",
+			},
 			Replicas:            initialize.Int32(1),
 			DataVolumeClaimSpec: testVolumeClaimSpec(),
 		}}
@@ -194,14 +200,20 @@ func TestCustomLabels(t *testing.T) {
 		cluster.ObjectMeta.Name = "instance-cluster"
 		cluster.ObjectMeta.Namespace = ns.Name
 		cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{
-			Name:                "max-instance",
+			Name: "max-instance",
+			InitContainer: &v1beta1.InitContainerSpec{
+				Image: "some-image",
+			},
 			Replicas:            initialize.Int32(1),
 			DataVolumeClaimSpec: testVolumeClaimSpec(),
 			Metadata: &v1beta1.Metadata{
 				Labels: map[string]string{"my.instance.label": "max"},
 			},
 		}, {
-			Name:                "lucy-instance",
+			Name: "lucy-instance",
+			InitContainer: &v1beta1.InitContainerSpec{
+				Image: "some-image",
+			},
 			Replicas:            initialize.Int32(1),
 			DataVolumeClaimSpec: testVolumeClaimSpec(),
 			Metadata: &v1beta1.Metadata{
@@ -396,11 +408,17 @@ func TestCustomAnnotations(t *testing.T) {
 		cluster.ObjectMeta.Name = "global-cluster"
 		cluster.ObjectMeta.Namespace = ns.Name
 		cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{
-			Name:                "daisy-instance1",
+			Name: "daisy-instance1",
+			InitContainer: &v1beta1.InitContainerSpec{
+				Image: "some-image",
+			},
 			Replicas:            initialize.Int32(1),
 			DataVolumeClaimSpec: testVolumeClaimSpec(),
 		}, {
-			Name:                "daisy-instance2",
+			Name: "daisy-instance2",
+			InitContainer: &v1beta1.InitContainerSpec{
+				Image: "some-image",
+			},
 			Replicas:            initialize.Int32(1),
 			DataVolumeClaimSpec: testVolumeClaimSpec(),
 		}}
@@ -447,14 +465,20 @@ func TestCustomAnnotations(t *testing.T) {
 		cluster.ObjectMeta.Name = "instance-cluster"
 		cluster.ObjectMeta.Namespace = ns.Name
 		cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{
-			Name:                "max-instance",
+			Name: "max-instance",
+			InitContainer: &v1beta1.InitContainerSpec{
+				Image: "some-image",
+			},
 			Replicas:            initialize.Int32(1),
 			DataVolumeClaimSpec: testVolumeClaimSpec(),
 			Metadata: &v1beta1.Metadata{
 				Annotations: map[string]string{"my.instance.annotation": "max"},
 			},
 		}, {
-			Name:                "lucy-instance",
+			Name: "lucy-instance",
+			InitContainer: &v1beta1.InitContainerSpec{
+				Image: "some-image",
+			},
 			Replicas:            initialize.Int32(1),
 			DataVolumeClaimSpec: testVolumeClaimSpec(),
 			Metadata: &v1beta1.Metadata{

deploy/cw-bundle.yaml

@@ -97,6 +97,8 @@ spec:
   image: postgres
   instances:
   - name: register-now
+    initContainer:
+      image: postgres
     dataVolumeClaimSpec:
       accessModes:
       - "ReadWriteMany"
@@ -233,6 +235,8 @@ spec:
   image: postgres
   instances:
   - name: samba
+    initContainer:
+      image: postgres
     dataVolumeClaimSpec:
       accessModes:
       - "ReadWriteMany"
@@ -438,6 +442,8 @@ spec:
   image: postgres
   instances:
   - name: samba
+    initContainer:
+      image: postgres
     dataVolumeClaimSpec:
       accessModes:
       - "ReadWriteMany"

e2e-tests/functions

@@ -116,7 +116,10 @@ func testCluster() *v1beta1.PostgresCluster {
 				Name: "myImagePullSecret"},
 			},
 			InstanceSets: []v1beta1.PostgresInstanceSetSpec{{
-				Name:                "instance1",
+				Name: "instance1",
+				InitContainer: &v1beta1.InitContainerSpec{
+					Image: "some-image",
+				},
 				Replicas:            initialize.Int32(1),
 				DataVolumeClaimSpec: testVolumeClaimSpec(),
 			}},

e2e-tests/tests/custom-extensions/02-assert.yaml

@@ -37,6 +37,7 @@ import (
 	"github.com/percona/percona-postgresql-operator/internal/pgbackrest"
 	"github.com/percona/percona-postgresql-operator/internal/pki"
 	"github.com/percona/percona-postgresql-operator/internal/postgres"
+	"github.com/percona/percona-postgresql-operator/percona/k8s"
 	"github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -1206,9 +1207,15 @@ func (r *Reconciler) reconcileInstance(
 				ctx, cluster, instanceCertificates, &instance.Spec.Template.Spec)
 		}
 
+		// K8SPG-708
+		initImage, err := k8s.InitImage(ctx, r.Client, cluster, spec)
+		if err != nil {
+			return errors.Wrap(err, "failed to determine initial init image")
+		}
+
 		err = patroni.InstancePod(
 			ctx, cluster, clusterConfigMap, clusterPodService, patroniLeaderService,
-			spec, instanceCertificates, instanceConfigMap, &instance.Spec.Template)
+			spec, instanceCertificates, instanceConfigMap, &instance.Spec.Template, initImage) // K8SPG-708
 	}
 
 	// Add pgMonitor resources to the instance Pod spec