K8SPG-260: Merge upstream (#860)

* K8SPG-260: Merge upstream

* remove golicense

* fix envtest

* fix crd

* fix e2e-tests

* fix envtest

* fix envtest (for real this time)

* fix cw-rbac

* fix scan build

* fix cw-rbac (for real this time)

* fix custom users secret

* fix demand-backup

* fix demand-backup

* fix tablespaces

* fix upgrade-minor

* fix major-upgrade

* fix unit tests

* fix upgrade-minor

* fix tests

* fix arm64 build

* fix golangci-lint errors

* use controller-gen v0.15.0

* fix golangci-lint errors

* fix misspell errors

* fix upgrade-minor

* fix upgrade-minor test

* remove images

* remove files in .github

* fix previous commit

* remove pgo files

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>
Co-authored-by: Andrii Dema <a.dema@jazzserve.com>

Author: 3 people

.github/workflows/reviewdog.yml

@@ -7,7 +7,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v5
         with:
-          go-version: '^1.20'
+          go-version: '^1.22'
       - uses: actions/checkout@v4
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '^1.20'
+          go-version: '^1.22'
       - run: go install mvdan.cc/sh/v3/cmd/shfmt@latest
       - run: $(go env GOPATH)/bin/shfmt -bn -ci -s -w .
       - name: suggester / shfmt

.github/workflows/scan.yml

@@ -28,7 +28,7 @@ jobs:
           export DOCKER_PUSH=0
           export DOCKER_SQUASH=0
           export DOCKER_DEFAULT_PLATFORM='linux/arm64'
-          ./e2e-tests/build
+          make build-docker-image
 
       - name: Run Trivy vulnerability scanner image (linux/arm64)
         uses: aquasecurity/trivy-action@0.24.0
@@ -46,7 +46,7 @@ jobs:
           export DOCKER_PUSH=0
           export DOCKER_SQUASH=0
           export DOCKER_DEFAULT_PLATFORM='linux/amd64'
-          ./e2e-tests/build
+          make build-docker-image
 
       - name: Run Trivy vulnerability scanner image (linux/amd64)
         uses: aquasecurity/trivy-action@0.24.0

.golangci.next.yaml

@@ -9,6 +9,7 @@ linters:
   disable-all: true
   enable:
     - contextcheck
+    - err113
     - errchkjson
     - gocritic
     - godot

.golangci.yaml

@@ -6,7 +6,6 @@ linters:
     - errchkjson
     - gci
     - gofumpt
-    - scopelint
   enable:
     - depguard
     - gomodguard
@@ -70,8 +69,6 @@ linters-settings:
         alias: $1$2
       - pkg: k8s.io/apimachinery/pkg/apis/(\w+)/(v[\w\d]+)
         alias: $1$2
-      - pkg: k8s.io/apimachinery/pkg/api/errors
-        alias: k8serrors
     no-unaliased: true
 
 run:

Jenkinsfile

@@ -247,8 +247,6 @@ repo_gpgcheck=0
 gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 EOF
         sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
-
-        curl -sL https://github.com/mitchellh/golicense/releases/download/v0.2.0/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
     """
 }
 
@@ -323,7 +321,7 @@ pipeline {
                                 export RELEASE=0
                                 export IMAGE=\$DOCKER_TAG
                                 docker buildx create --use
-                                ./e2e-tests/build
+                                make build-docker-image
                                 docker logout
                             "
                         sudo rm -rf ./build
@@ -363,35 +361,6 @@ pipeline {
                          """
                      }
                  }
-                 stage('GoLicense test') {
-                     steps {
-                         sh '''
-                             mkdir -p $WORKSPACE/src/github.com/percona
-                             ln -s $WORKSPACE $WORKSPACE/src/github.com/percona/percona-postgresql-operator
-                             sg docker -c "
-                                 docker run \
-                                     --rm \
-                                     -v $WORKSPACE/src/github.com/percona/percona-postgresql-operator:/go/src/github.com/percona/percona-postgresql-operator \
-                                     -w /go/src/github.com/percona/percona-postgresql-operator \
-                                     -e GO111MODULE=on \
-                                     -e GOFLAGS='-buildvcs=false' \
-                                     golang:1.22 sh -c 'go build -v -o percona-postgresql-operator github.com/percona/percona-postgresql-operator/cmd/postgres-operator'
-                             "
-                         '''
-
-                         withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'GITHUB_TOKEN')]) {
-                             sh """
-                                 golicense -plain ./percona-postgresql-operator \
-                                     | grep -v 'license not found' \
-                                     | sed -r 's/^[^ ]+[ ]+//' \
-                                     | sort \
-                                     | uniq \
-                                     > golicense-new || true
-                                 diff -u ./e2e-tests/license/compare/golicense golicense-new
-                             """
-                         }
-                     }
-                 }
              }
         }
         stage('Run E2E tests') {

LICENSE.md

@@ -177,7 +177,7 @@
    END OF TERMS AND CONDITIONS
 
    Copyright 2017 - 2024 Crunchy Data Solutions, Inc.
-   Copyright 2021 Percona, LLC 
+   Copyright 2021 - 2024 Percona, LLC 
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -12,19 +12,12 @@ QUERIES_CONFIG_DIR ?= hack/tools/queries
 # Buildah's "build" used to be "bud". Use the alias to be compatible for a while.
 BUILDAH_BUILD ?= buildah bud
 
-DEBUG_BUILD ?= false
-GO ?= go
-GO_BUILD = $(GO_CMD) build -trimpath
-GO_CMD = $(GO_ENV) $(GO)
+GO ?= CGO_ENABLED=1 go
+GO_BUILD = $(GO) build -trimpath
 GO_TEST ?= $(GO) test
 KUTTL ?= kubectl-kuttl
 KUTTL_TEST ?= $(KUTTL) test
 
-# Disable optimizations if creating a debug build
-ifeq ("$(DEBUG_BUILD)", "true")
-	GO_BUILD = $(GO_CMD) build -gcflags='all=-N -l'
-endif
-
 ##@ General
 
 # The help target prints out all targets with their descriptions organized
@@ -191,7 +184,8 @@ build-postgres-operator-image: build/postgres-operator/Dockerfile
 ##@ Test
 .PHONY: check
 check: ## Run basic go tests with coverage output
-	$(GO_TEST) -cover ./...
+check: get-pgmonitor
+	QUERIES_CONFIG_DIR="$(CURDIR)/${QUERIES_CONFIG_DIR}" $(GO_TEST) -cover ./...
 
 # Available versions: curl -s 'https://storage.googleapis.com/kubebuilder-tools/' | grep -o '<Key>[^<]*</Key>'
 # - KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT=true
@@ -206,7 +200,7 @@ check-envtest: get-pgmonitor
 		$(GO_TEST) -count=1 -cover -tags=envtest ./...
 
 # The "PGO_TEST_TIMEOUT_SCALE" environment variable (default: 1) can be set to a
-# positive number that extends test timeouts. The following runs tests with
+# positive number that extends test timeouts. The following runs tests with 
 # timeouts that are 20% longer than normal:
 # make check-envtest-existing PGO_TEST_TIMEOUT_SCALE=1.2
 .PHONY: check-envtest-existing
@@ -230,7 +224,7 @@ generate-kuttl: export KUTTL_PG_UPGRADE_FROM_VERSION ?= 15
 generate-kuttl: export KUTTL_PG_UPGRADE_TO_VERSION ?= 16
 generate-kuttl: export KUTTL_PG_VERSION ?= 16
 generate-kuttl: export KUTTL_POSTGIS_VERSION ?= 3.4
-generate-kuttl: export KUTTL_PSQL_IMAGE ?= registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.2-0
+generate-kuttl: export KUTTL_PSQL_IMAGE ?= registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.3-1
 generate-kuttl: export KUTTL_TEST_DELETE_NAMESPACE ?= kuttl-test-delete-namespace
 generate-kuttl: ## Generate kuttl tests
 	[ ! -d testing/kuttl/e2e-generated ] || rm -r testing/kuttl/e2e-generated
@@ -242,7 +236,6 @@ generate-kuttl: ## Generate kuttl tests
 	14 ) export KUTTL_BITNAMI_IMAGE_TAG=14.5.0-debian-11-r37 ;; \
 	13 ) export KUTTL_BITNAMI_IMAGE_TAG=13.8.0-debian-11-r39 ;; \
 	12 ) export KUTTL_BITNAMI_IMAGE_TAG=12.12.0-debian-11-r40 ;; \
-	11 ) export KUTTL_BITNAMI_IMAGE_TAG=11.17.0-debian-11-r39 ;; \
 	esac; \
 	render() { envsubst '"'"' \
 		$$KUTTL_PG_UPGRADE_FROM_VERSION $$KUTTL_PG_UPGRADE_TO_VERSION \
@@ -346,7 +339,6 @@ generate-cw-bundle:
 	$(KUSTOMIZE) build ./config/cw-bundle/ > ./deploy/cw-bundle.yaml
 
 
-
 ##@ Release
 
 .PHONY: license licenses
@@ -380,17 +372,12 @@ KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
 	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.3)
 
-# Available versions: curl -s 'https://storage.googleapis.com/kubebuilder-tools/' | grep -o '<Key>[^<]*</Key>'
-# - ENVTEST_K8S_VERSION=1.19.2
-hack/tools/envtest: SHELL = bash
-hack/tools/envtest:
-	source '$(shell $(GO) list -f '{{ .Dir }}' -m 'sigs.k8s.io/controller-runtime')/hack/setup-envtest.sh' && fetch_envtest_tools $@
-
-ENVTEST = $(shell pwd)/bin/setup-envtest
-envtest: ## Download envtest-setup locally if necessary.
+ENVTEST ?= hack/tools/setup-envtest
+tools: tools/setup-envtest
+tools/setup-envtest:
 	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
 
-build-docker-image:
+build-docker-image: get-pgmonitor
 	ROOT_REPO=$(ROOT_REPO) VERSION=$(VERSION) IMAGE=$(IMAGE) $(ROOT_REPO)/e2e-tests/build
 
 build-extension-installer-image:

build/crd/crunchy/generated/postgres-operator.crunchydata.com_pgupgrades.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.15.0
   name: pgupgrades.postgres-operator.crunchydata.com
 spec:
   group: postgres-operator.crunchydata.com

build/crd/crunchy/generated/postgres-operator.crunchydata.com_postgresclusters.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.15.0
   name: postgresclusters.postgres-operator.crunchydata.com
 spec:
   group: postgres-operator.crunchydata.com
@@ -13863,7 +13863,7 @@ spec:
                       Patroni dynamic configuration settings. Changes to this value will be
                       automatically reloaded without validation. Changes to certain PostgreSQL
                       parameters cause PostgreSQL to restart.
-                      More info: https://patroni.readthedocs.io/en/latest/SETTINGS.html
+                      More info: https://patroni.readthedocs.io/en/latest/dynamic_configuration.html
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
                   leaderLeaseDurationSeconds:
@@ -17079,6 +17079,18 @@ spec:
                       service:
                         description: Specification of the service that exposes PgBouncer.
                         properties:
+                          externalTrafficPolicy:
+                            description: 'More info: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-policies'
+                            enum:
+                            - Cluster
+                            - Local
+                            type: string
+                          internalTrafficPolicy:
+                            description: 'More info: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-policies'
+                            enum:
+                            - Cluster
+                            - Local
+                            type: string
                           loadBalancerSourceRanges:
                             description: |-
                               LoadBalancerSourceRanges is a list of IP CIDRs allowed access to load.
@@ -17413,6 +17425,18 @@ spec:
                 description: Specification of the service that exposes PostgreSQL
                   replica instances
                 properties:
+                  externalTrafficPolicy:
+                    description: 'More info: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-policies'
+                    enum:
+                    - Cluster
+                    - Local
+                    type: string
+                  internalTrafficPolicy:
+                    description: 'More info: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-policies'
+                    enum:
+                    - Cluster
+                    - Local
+                    type: string
                   loadBalancerSourceRanges:
                     description: |-
                       LoadBalancerSourceRanges is a list of IP CIDRs allowed access to load.
@@ -17453,6 +17477,18 @@ spec:
                 description: Specification of the service that exposes the PostgreSQL
                   primary instance.
                 properties:
+                  externalTrafficPolicy:
+                    description: 'More info: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-policies'
+                    enum:
+                    - Cluster
+                    - Local
+                    type: string
+                  internalTrafficPolicy:
+                    description: 'More info: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-policies'
+                    enum:
+                    - Cluster
+                    - Local
+                    type: string
                   loadBalancerSourceRanges:
                     description: |-
                       LoadBalancerSourceRanges is a list of IP CIDRs allowed access to load.
@@ -19143,6 +19179,18 @@ spec:
                       service:
                         description: Specification of the service that exposes pgAdmin.
                         properties:
+                          externalTrafficPolicy:
+                            description: 'More info: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-policies'
+                            enum:
+                            - Cluster
+                            - Local
+                            type: string
+                          internalTrafficPolicy:
+                            description: 'More info: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-policies'
+                            enum:
+                            - Cluster
+                            - Local
+                            type: string
                           loadBalancerSourceRanges:
                             description: |-
                               LoadBalancerSourceRanges is a list of IP CIDRs allowed access to load.
@@ -19444,6 +19492,7 @@ spec:
                         ALTER ROLE options except for PASSWORD. This field is ignored for the
                         "postgres" user.
                         More info: https://www.postgresql.org/docs/current/role-attributes.html
+                      maxLength: 200
                       pattern: ^[^;]*$
                       type: string
                     password:
@@ -19472,6 +19521,7 @@ spec:
                   required:
                   - name
                   type: object
+                maxItems: 64
                 type: array
                 x-kubernetes-list-map-keys:
                 - name
@@ -19572,6 +19622,11 @@ spec:
                 description: Current state of PostgreSQL instances.
                 items:
                   properties:
+                    desiredPGDataVolume:
+                      additionalProperties:
+                        type: string
+                      description: Desired Size of the pgData volume
+                      type: object
                     name:
                       type: string
                     readyReplicas:
@@ -19844,8 +19899,6 @@ spec:
                     type: object
                 type: object
               registrationRequired:
-                description: Version information for installations with a registration
-                  requirement.
                 properties:
                   pgoVersion:
                     type: string
@@ -19859,8 +19912,6 @@ spec:
                 description: The instance set associated with the startupInstance
                 type: string
               tokenRequired:
-                description: Signals the need for a token to be applied when registration
-                  is required.
                 type: string
               userInterface:
                 description: Current state of the PostgreSQL user interface.

build/crd/crunchybridgeclusters/kustomization.yaml

@@ -5,18 +5,6 @@ resources:
 - generated/postgres-operator.crunchydata.com_crunchybridgeclusters.yaml
 
 patches:
-# Remove the zero status field included by controller-gen@v0.8.0. These zero
-# values conflict with the CRD controller in Kubernetes before v1.22.
-# - https://github.com/kubernetes-sigs/controller-tools/pull/630
-# - https://pr.k8s.io/100970
-#- target:
-#    group: apiextensions.k8s.io
-#    version: v1
-#    kind: CustomResourceDefinition
-#    name: crunchybridgeclusters.postgres-operator.crunchydata.com
-#  patch: |-
-#    - op: remove
-#      path: /status
 - target:
     group: apiextensions.k8s.io
     version: v1
@@ -29,9 +17,3 @@ patches:
       value:
         app.kubernetes.io/name: pgo
         app.kubernetes.io/version: latest
-- target:
-    group: apiextensions.k8s.io
-    version: v1
-    kind: CustomResourceDefinition
-    name: crunchybridgeclusters.postgres-operator.crunchydata.com
-  path: immutable.yaml