some adjust

Author: yanboer

platforms/kubernetes/postgres-operator/deploy/postgres-operator.yaml.template

@@ -357,50 +357,51 @@ spec:
                 type: object
                 x-kubernetes-preserve-unknown-fields: true
                 properties:
-                  backups3:
+                  backupToS3:
                     type: object
+                    x-kubernetes-preserve-unknown-fields: true
                     properties:
                       name:
                         type: string
-                  manual:
-                    type: object
-                    properties:
-                      trigger-id:
-                        type: integer
-                  cron:
-                    type: object
-                    properties:
-                      enable:
-                        type: boolean
-                      schedule:
-                        type: string
-                  policy:
-                    type: object
-                    properties:
-                      archive:
-                        type: string
-                        enum:
-                        - 'on'
-                        - 'off'
-                        default: 'off'
-                      compression:
-                        type: string
-                        enum:
-                        - 'gzip'
-                        - 'bzip2'
-                        - 'snappy'
-                        - 'none'
-                        default: 'none'
-                      encryption:
-                        type: string
-                        enum:
-                        - 'AES256'
-                        - 'aws:kms'
-                        - 'none'
-                        default: 'none'
-                      retention:
-                        type: string
-                        default: 'none'
+                      manual:
+                        type: object
+                        properties:
+                          trigger-id:
+                            type: integer
+                      cron:
+                        type: object
+                        properties:
+                          enable:
+                            type: boolean
+                          schedule:
+                            type: string
+                      policy:
+                        type: object
+                        properties:
+                          archive:
+                            type: string
+                            enum:
+                            - 'on'
+                            - 'off'
+                            default: 'off'
+                          compression:
+                            type: string
+                            enum:
+                            - 'gzip'
+                            - 'bzip2'
+                            - 'snappy'
+                            - 'none'
+                            default: 'none'
+                          encryption:
+                            type: string
+                            enum:
+                            - 'AES256'
+                            - 'aws:kms'
+                            - 'none'
+                            default: 'none'
+                          retention:
+                            type: string
+                            default: 'none'
               restore:
                 type: object
                 x-kubernetes-preserve-unknown-fields: true

platforms/kubernetes/postgres-operator/deploy/postgresql.yaml

@@ -165,27 +165,27 @@ spec:
     BUCKET: "postgresql"
     PATH: "test/backup"
 #  backupCluster:
-##    backups3:
+#    backupToS3:
 ##      name: postgresql-backup
-#    manual:
-#      trigger-id: 2
-#    cron:
-#      enable: true                          # whether to enable scheduled backup
-#      schedule: "0 1 * * wed,sat"             # cron expression: minute(0-59) hour(0-23) day(1-31) month(1-12) day_of_week(0-6, mon,tue,wed,thu,fri,sat,sun)
-#                                                   execute once a day at 23:00: 0 23 * * *
-#                                                   1:00 every weekend: 0 1 * * sun
-#                                                   executed at 1:00 AM on the first day of each month : 0 1 1 * *
-#                                                   execute once every day at 0:00, 13:00: 0 0,13 * * *
-#                                                   executed at 1:00 AM every Wednesday and Saturday: 0 1 * * wed,sat
-#                                               For more information see https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html?highlight=cron
-#    policy:                                  # take effect on next backup
-#      archive: 'on'                            # whether to archive WAL. on/off
-#      compression: gzip                        # backup compression. none/gzip/bzip2/snappy
-#      encryption: AES256                       # backup encryption. none/AES256/aws:kms
-#      retention: REDUNDANCY 10                 # backup retention policy. none/REDUNDANCY value/RECOVERY WINDOW OF value {DAYS | WEEKS | MONTHS}
-                                                 # none: backup does not expire
-                                                 # REDUNDANCY value: keep the number of backups. eg: REDUNDANCY 10.
-                                                 # RECOVERY WINDOW OF value {DAYS | WEEKS | MONTHS}: keep backup time. eg: RECOVERY WINDOW OF 60 DAYS
+#      manual:
+#        trigger-id: 1
+#      cron:
+#        enable: true                          # whether to enable scheduled backup
+#        schedule: "0 1 * * wed,sat"             # cron expression: minute(0-59) hour(0-23) day(1-31) month(1-12) day_of_week(0-6, mon,tue,wed,thu,fri,sat,sun)
+##                                                     execute once a day at 23:00: 0 23 * * *
+##                                                     1:00 every weekend: 0 1 * * sun
+##                                                     executed at 1:00 AM on the first day of each month : 0 1 1 * *
+##                                                     execute once every day at 0:00, 13:00: 0 0,13 * * *
+##                                                     executed at 1:00 AM every Wednesday and Saturday: 0 1 * * wed,sat
+##                                                 For more information see https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html?highlight=cron
+#      policy:                                  # take effect on next backup
+#        archive: 'on'                            # whether to archive WAL. on/off
+#        compression: gzip                        # backup compression. none/gzip/bzip2/snappy
+#        encryption: AES256                       # backup encryption. none/AES256/aws:kms
+#        retention: REDUNDANCY 10                 # backup retention policy. none/REDUNDANCY value/RECOVERY WINDOW OF value {DAYS | WEEKS | MONTHS}
+                                                   # none: backup does not expire
+                                                   # REDUNDANCY value: keep the number of backups. eg: REDUNDANCY 10.
+                                                   # RECOVERY WINDOW OF value {DAYS | WEEKS | MONTHS}: keep backup time. eg: RECOVERY WINDOW OF 60 DAYS
 #  restore:
 #    froms3:
 ##      name: postgresql-backup

platforms/kubernetes/postgres-operator/postgres/constants.py

@@ -49,23 +49,23 @@
 SPEC_POSTGRESQL_USERS_NORMAL = "normal"
 SPEC_POSTGRESQL_USERS_USER_PASSWORD = "password"
 SPEC_POSTGRESQL_USERS_USER_NAME = "name"
-SPEC_BACKUP = "backupCluster"
-SPEC_BACKUP_BACKUPS3 = "backups3"
-SPEC_BACKUP_BACKUPS3_NAME = "name"
-SPEC_BACKUP_MANUAL = "manual"
-SPEC_BACKUP_TRIGGER_ID = "trigger-id"
-SPEC_BACKUP_CRON = "cron"
-SPEC_BACKUP_CRON_ENABLE = "enable"
-SPEC_BACKUP_CRON_SCHEDULE = "schedule"
-SPEC_BACKUP_POLICY = "policy"
-SPEC_BACKUP_POLICY_ARCHIVE = "archive"
-SPEC_BACKUP_POLICY_ARCHIVE_DEFAULT_VALUE = "off"
-SPEC_BACKUP_POLICY_COMPRESSION = "compression"
-SPEC_BACKUP_POLICY_COMPRESSION_DEFAULT_VALUE = "none"
-SPEC_BACKUP_POLICY_ENCRYPTION = "encryption"
-SPEC_BACKUP_POLICY_ENCRYPTION_DEFAULT_VALUE = "none"
-SPEC_BACKUP_POLICY_RETENTION = "retention"
-SPEC_BACKUP_POLICY_RETENTION_DEFAULT_VALUE = "none"
+SPEC_BACKUPCLUSTER = "backupCluster"
+SPEC_BACKUPTOS3 = "backupToS3"
+SPEC_BACKUPTOS3_NAME = "name"
+SPEC_BACKUPTOS3_MANUAL = "manual"
+SPEC_BACKUPTOS3_MANUAL_TRIGGER_ID = "trigger-id"
+SPEC_BACKUPTOS3_CRON = "cron"
+SPEC_BACKUPTOS3_CRON_ENABLE = "enable"
+SPEC_BACKUPTOS3_CRON_SCHEDULE = "schedule"
+SPEC_BACKUPTOS3_POLICY = "policy"
+SPEC_BACKUPTOS3_POLICY_ARCHIVE = "archive"
+SPEC_BACKUPTOS3_POLICY_ARCHIVE_DEFAULT_VALUE = "off"
+SPEC_BACKUPTOS3_POLICY_COMPRESSION = "compression"
+SPEC_BACKUPTOS3_POLICY_COMPRESSION_DEFAULT_VALUE = "none"
+SPEC_BACKUPTOS3_POLICY_ENCRYPTION = "encryption"
+SPEC_BACKUPTOS3_POLICY_ENCRYPTION_DEFAULT_VALUE = "none"
+SPEC_BACKUPTOS3_POLICY_RETENTION = "retention"
+SPEC_BACKUPTOS3_POLICY_RETENTION_DEFAULT_VALUE = "none"
 RESTORE = "restore"
 RESTORE_FROMSSH = "fromssh"
 RESTORE_FROMSSH_PATH = "path"

platforms/kubernetes/postgres-operator/postgres/handle.py

@@ -152,23 +152,23 @@
     SPEC_S3_ENDPOINT,
     SPEC_S3_BUCKET,
     SPEC_S3_PATH,
-    SPEC_BACKUP,
-    SPEC_BACKUP_BACKUPS3,
-    SPEC_BACKUP_BACKUPS3_NAME,
-    SPEC_BACKUP_MANUAL,
-    SPEC_BACKUP_TRIGGER_ID,
-    SPEC_BACKUP_CRON,
-    SPEC_BACKUP_CRON_ENABLE,
-    SPEC_BACKUP_CRON_SCHEDULE,
-    SPEC_BACKUP_POLICY,
-    SPEC_BACKUP_POLICY_ARCHIVE,
-    SPEC_BACKUP_POLICY_ARCHIVE_DEFAULT_VALUE,
-    SPEC_BACKUP_POLICY_COMPRESSION,
-    SPEC_BACKUP_POLICY_COMPRESSION_DEFAULT_VALUE,
-    SPEC_BACKUP_POLICY_ENCRYPTION,
-    SPEC_BACKUP_POLICY_ENCRYPTION_DEFAULT_VALUE,
-    SPEC_BACKUP_POLICY_RETENTION,
-    SPEC_BACKUP_POLICY_RETENTION_DEFAULT_VALUE,
+    SPEC_BACKUPCLUSTER,
+    SPEC_BACKUPTOS3,
+    SPEC_BACKUPTOS3_NAME,
+    SPEC_BACKUPTOS3_MANUAL,
+    SPEC_BACKUPTOS3_MANUAL_TRIGGER_ID,
+    SPEC_BACKUPTOS3_CRON,
+    SPEC_BACKUPTOS3_CRON_ENABLE,
+    SPEC_BACKUPTOS3_CRON_SCHEDULE,
+    SPEC_BACKUPTOS3_POLICY,
+    SPEC_BACKUPTOS3_POLICY_ARCHIVE,
+    SPEC_BACKUPTOS3_POLICY_ARCHIVE_DEFAULT_VALUE,
+    SPEC_BACKUPTOS3_POLICY_COMPRESSION,
+    SPEC_BACKUPTOS3_POLICY_COMPRESSION_DEFAULT_VALUE,
+    SPEC_BACKUPTOS3_POLICY_ENCRYPTION,
+    SPEC_BACKUPTOS3_POLICY_ENCRYPTION_DEFAULT_VALUE,
+    SPEC_BACKUPTOS3_POLICY_RETENTION,
+    SPEC_BACKUPTOS3_POLICY_RETENTION_DEFAULT_VALUE,
     RESTORE_FROMS3,
     RESTORE_FROMS3_NAME,
     RESTORE_FROMS3_RECOVERY,
@@ -224,8 +224,8 @@
                               VOLUMECLAIMTEMPLATES)
 DIFF_FIELD_SPEC_ANTIAFFINITY = (SPEC, SPEC_ANTIAFFINITY)
 DIFF_FIELD_SPEC_S3 = (SPEC, SPEC_S3)
-DIFF_FIELD_SPEC_BACKUP = (SPEC, SPEC_BACKUP)
-DIFF_FIELD_SPEC_BACKUP_MANUAL = (SPEC, SPEC_BACKUP, SPEC_BACKUP_MANUAL)
+DIFF_FIELD_SPEC_BACKUPS3 = (SPEC, SPEC_BACKUPCLUSTER, SPEC_BACKUPTOS3)
+DIFF_FIELD_SPEC_BACKUPS3_MANUAL = (SPEC, SPEC_BACKUPCLUSTER, SPEC_BACKUPTOS3, SPEC_BACKUPTOS3_MANUAL)
 STATEFULSET_REPLICAS = 1
 PG_CONFIG_MASTER_LARGE_THAN_SLAVE = ("max_connections", "max_worker_processes", "max_wal_senders", "max_prepared_transactions", "max_locks_per_transaction")
 PG_CONFIG_IGNORE = ("block_size", "data_checksums", "data_directory_mode",
@@ -1648,10 +1648,10 @@ def get_policy_env(policy: TypedDict) -> List:
     res = list()
 
     default_policy = {
-        SPEC_BACKUP_POLICY_ARCHIVE: SPEC_BACKUP_POLICY_ARCHIVE_DEFAULT_VALUE,
-        SPEC_BACKUP_POLICY_COMPRESSION: SPEC_BACKUP_POLICY_COMPRESSION_DEFAULT_VALUE,
-        SPEC_BACKUP_POLICY_ENCRYPTION: SPEC_BACKUP_POLICY_ENCRYPTION_DEFAULT_VALUE,
-        SPEC_BACKUP_POLICY_RETENTION: SPEC_BACKUP_POLICY_RETENTION_DEFAULT_VALUE
+        SPEC_BACKUPTOS3_POLICY_ARCHIVE: SPEC_BACKUPTOS3_POLICY_ARCHIVE_DEFAULT_VALUE,
+        SPEC_BACKUPTOS3_POLICY_COMPRESSION: SPEC_BACKUPTOS3_POLICY_COMPRESSION_DEFAULT_VALUE,
+        SPEC_BACKUPTOS3_POLICY_ENCRYPTION: SPEC_BACKUPTOS3_POLICY_ENCRYPTION_DEFAULT_VALUE,
+        SPEC_BACKUPTOS3_POLICY_RETENTION: SPEC_BACKUPTOS3_POLICY_RETENTION_DEFAULT_VALUE
     }
 
     for k, v in policy.items():
@@ -1845,7 +1845,10 @@ def is_backup_mode(
     status: kopf.Status,
     logger: logging.Logger,
 ) -> bool:
-    if spec.get(SPEC_BACKUP) == None:
+    if spec.get(SPEC_BACKUPCLUSTER) == None:
+        return False
+
+    if spec[SPEC_BACKUPCLUSTER].get(SPEC_BACKUPTOS3) == None:
         return False
 
     return True
@@ -1865,7 +1868,7 @@ def is_s3_manual_backup_mode(
         logger.warning("s3 related information is not set.")
         return False
 
-    if spec[SPEC_BACKUP].get(SPEC_BACKUP_MANUAL) != None:
+    if spec[SPEC_BACKUPCLUSTER].get(SPEC_BACKUPTOS3, {}).get(SPEC_BACKUPTOS3_MANUAL) != None:
         return True
 
     return False
@@ -1885,7 +1888,7 @@ def is_s3_cron_backup_mode(
         logger.warning("s3 related information is not set.")
         return False
 
-    if spec[SPEC_BACKUP].get(SPEC_BACKUP_CRON) != None:
+    if spec[SPEC_BACKUPCLUSTER].get(SPEC_BACKUPTOS3, {}).get(SPEC_BACKUPTOS3_CRON) != None:
         return True
 
     return False
@@ -1923,10 +1926,10 @@ def backup_postgresql_to_s3(
     s3 = spec[SPEC_S3].copy()
     s3_list = get_s3_env(s3)
 
-    backup_policy = spec.get(SPEC_BACKUP, {}).get(SPEC_BACKUP_POLICY, {})
+    backup_policy = spec[SPEC_BACKUPCLUSTER][SPEC_BACKUPTOS3].get(SPEC_BACKUPTOS3_POLICY, {})
     policy_list = get_policy_env(backup_policy)
 
-    name = spec[SPEC_BACKUP].get(SPEC_BACKUP_BACKUPS3, {}).get(SPEC_BACKUP_BACKUPS3_NAME, None)
+    name = spec[SPEC_BACKUPCLUSTER][SPEC_BACKUPTOS3].get(SPEC_BACKUPTOS3_NAME, None)
     name_list = get_backup_name_env(meta, name)
 
     s3_info = [*s3_list, *policy_list, *name_list]
@@ -3575,7 +3578,7 @@ def correct_backup_status(
     logger: logging.Logger,
 ) -> None:
     if is_s3_manual_backup_mode(meta, spec, patch, status, logger) or is_s3_cron_backup_mode(meta, spec, patch, status, logger):
-        if spec.get(SPEC_BACKUP, {}).get(SPEC_BACKUP_POLICY, {}).get(SPEC_BACKUP_POLICY_ARCHIVE, "") == "on":
+        if spec[SPEC_BACKUPCLUSTER][SPEC_BACKUPTOS3].get(SPEC_BACKUPTOS3_POLICY, {}).get(SPEC_BACKUPTOS3_POLICY_ARCHIVE, "") == "on":
             readwrite_conns = connections(spec, meta, patch,
                                           get_field(POSTGRESQL, READWRITEINSTANCE),
                                           False, None, logger, None, status, False)
@@ -4856,11 +4859,11 @@ def update_cluster(
             update_configs(meta, spec, patch, status, logger, AC, FIELD, OLD,
                            NEW)
 
-            if FIELD[0:len(DIFF_FIELD_SPEC_BACKUP_MANUAL
-                           )] == DIFF_FIELD_SPEC_BACKUP_MANUAL or (
-                               FIELD[0:len(DIFF_FIELD_SPEC_BACKUP)]
-                               == DIFF_FIELD_SPEC_BACKUP and AC == "add"
-                               and OLD is None and SPEC_BACKUP_MANUAL in NEW):
+            if FIELD[0:len(DIFF_FIELD_SPEC_BACKUPS3_MANUAL
+                           )] == DIFF_FIELD_SPEC_BACKUPS3_MANUAL or (
+                    FIELD[0:len(DIFF_FIELD_SPEC_BACKUPS3)]
+                    == DIFF_FIELD_SPEC_BACKUPS3 and AC == "add"
+                    and OLD is None and SPEC_BACKUPTOS3_MANUAL in NEW):
                 need_backup_cluster = True
 
 
@@ -4922,8 +4925,9 @@ def cron_cluster(
     scheduler: BackgroundScheduler,
 ) -> None:
 
-    new_cron_expression = spec.get(SPEC_BACKUP, {}).get(SPEC_BACKUP_CRON, {}).get(SPEC_BACKUP_CRON_SCHEDULE, None)
-    cron_enable = spec.get(SPEC_BACKUP, {}).get(SPEC_BACKUP_CRON, {}).get(SPEC_BACKUP_CRON_ENABLE, None)
+    cron = spec.get(SPEC_BACKUPCLUSTER, {}).get(SPEC_BACKUPTOS3, {}).get(SPEC_BACKUPTOS3_CRON, {})
+    new_cron_expression = cron.get(SPEC_BACKUPTOS3_CRON_SCHEDULE, None)
+    cron_enable = cron.get(SPEC_BACKUPTOS3_CRON_ENABLE, None)
     jobs = scheduler.get_jobs()
 
     if cron_enable:

platforms/kubernetes/postgres-operator/postgres/postgres.py

@@ -13,9 +13,10 @@
 from kubernetes import client, config
 
 from constants import (
-    SPEC_BACKUP,
-    SPEC_BACKUP_CRON,
-    SPEC_BACKUP_CRON_ENABLE,
+    SPEC_BACKUPCLUSTER,
+    SPEC_BACKUPTOS3,
+    SPEC_BACKUPTOS3_CRON,
+    SPEC_BACKUPTOS3_CRON_ENABLE,
 )
 
 
@@ -126,7 +127,7 @@ def cluster_timer(
     RESOURCE_POSTGRESQL,
     backoff=operator_config.BOOTSTRAP_RETRY_DELAY,
     initial_delay=30,
-    when=lambda spec, **_: spec.get(SPEC_BACKUP, {}).get(SPEC_BACKUP_CRON, {}).get(SPEC_BACKUP_CRON_ENABLE, False) is True,
+    when=lambda spec, **_: spec.get(SPEC_BACKUPCLUSTER, {}).get(SPEC_BACKUPTOS3, {}).get(SPEC_BACKUPTOS3_CRON, {}).get(SPEC_BACKUPTOS3_CRON_ENABLE, False) is True,
 )
 def cluster_daemon(
     meta: kopf.Meta,