support citus v11.1.3(pg-13/14/15) v10.2.8(pg-12)

Author: zlianzhuang

image/postgresql/Dockerfile.template

@@ -32,11 +32,13 @@ RUN apt install -y --no-install-recommends \
 		libxslt-dev \
 		uuid-dev \
 		tcl-dev \
-{{ if .major >= 14 then ( -}}
         liblz4-dev \
-{{ ) else "" end -}}
 		libperl-dev \
-		libkrb5-dev
+		libkrb5-dev \
+		libcurl4-openssl-dev \
+		libzstd-dev \
+		automake \
+		autoconf
 #{{ if env.arch == "aarch64"  then ( -}}
 #        ""
 #{{ ) else "" end -}}
@@ -84,6 +86,13 @@ RUN	cd /usr/src/pg_dirtyread; \
 	make PG_CONFIG=/var/lib/postgresql/bin/pg_config -j `expr $(nproc) - 2` >/dev/null; \
 	make install PG_CONFIG=/var/lib/postgresql/bin/pg_config;
 
+ADD citus /usr/src/citus
+RUN	cd /usr/src/citus; \
+	git checkout {{ .citus }}; \
+	PG_CONFIG=/var/lib/postgresql/bin/pg_config ./configure ; \
+	make PG_CONFIG=/var/lib/postgresql/bin/pg_config -j `expr $(nproc) - 2` >/dev/null; \
+	make install PG_CONFIG=/var/lib/postgresql/bin/pg_config;
+
 RUN cd /var/lib/postgresql; \
 		tar czhvf lib.tar.gz `find . -type f | xargs ldd 2>/dev/null | grep "=>" | cut -d ">" -f 2 | cut -d "(" -f 1 | sort -u`
 
@@ -95,6 +104,8 @@ ENV DATA /var/lib/postgresql/data
 ENV XDG_CONFIG_HOME /var/lib/postgresql/data/auto_failover
 ENV XDG_DATA_HOME /var/lib/postgresql/data/auto_failover
 ENV ASSIST /var/lib/postgresql/data/assist
+ENV PGPASSFILE /var/lib/postgresql/data/assist/pgpassfile
+ENV LD_LIBRARY_PATH /var/lib/postgresql/lib/
 ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/postgresql/bin
 ENV PGHOME /var/lib/postgresql
 VOLUME /var/lib/postgresql/data

image/postgresql/generate_image.sh

@@ -42,6 +42,7 @@ build_image()
 	cp -r ../../source_code/postgresql "$dir/"
 	cp -r ../../source_code/pg_auto_failover "$dir/"
 	cp -r ../../source_code/pg_dirtyread "$dir/"
+	cp -r ../../source_code/citus "$dir/"
 
 	cd "$dir"
 	echo "build docker image $image ..."

pgversions.json

@@ -6,6 +6,7 @@
     "tag": ["REL_12_12"],
     "failover": "v1.6.4",
     "dirtyread": "2.4",
+    "citus": "v10.2.8",
     "name": "radondb-postgresql",
     "version": "v1.0.0",
     "arches": [
@@ -20,6 +21,7 @@
     "tag": ["REL_13_8"],
     "failover": "v1.6.4",
     "dirtyread": "2.4",
+    "citus": "v11.1.3",
     "name": "radondb-postgresql",
     "version": "v1.0.0",
     "arches": [
@@ -34,6 +36,7 @@
     "tag": ["REL_14_5"],
     "failover": "v1.6.4",
     "dirtyread": "2.4",
+    "citus": "v11.1.3",
     "name": "radondb-postgresql",
     "version": "v1.0.0",
     "arches": [
@@ -48,6 +51,7 @@
     "tag": ["REL_15_0"],
     "failover": "v2.0",
     "dirtyread": "2.4",
+    "citus": "v11.1.3",
     "name": "radondb-postgresql",
     "version": "v1.0.0",
     "arches": [

platforms/kubernetes/postgres-operator/postgres/handle.py

@@ -150,6 +150,7 @@
 DIFF_ADD = "add"
 DIFF_CHANGE = "change"
 DIFF_REMOVE = "remove"
+PGPASSFILE_PATH = ASSIST_DIR + "/pgpassfile"
 DIFF_FIELD_ACTION = (SPEC, ACTION)
 DIFF_FIELD_SERVICE = (SPEC, SERVICES)
 DIFF_FIELD_AUTOFAILOVER_HBAS = (SPEC, AUTOFAILOVER, HBAS)
@@ -927,6 +928,7 @@ def create_postgresql(
                 CONTAINER_ENV_VALUE: value
             })
     if mode == MACHINE_MODE:
+        machine_env += PG_CONFIG_PREFIX + "shared_preload_libraries='citus,pgautofailover,pg_stat_statements'" + "\n"
         machine_env += PG_CONFIG_PREFIX + 'log_truncate_on_rotation=true' + "\n"
         machine_env += PG_CONFIG_PREFIX + 'logging_collector=on' + "\n"
         machine_env += PG_CONFIG_PREFIX + "log_directory='log'" + "\n"
@@ -941,6 +943,10 @@ def create_postgresql(
         machine_env += PG_CONFIG_PREFIX + "tcp_keepalives_interval=30" + "\n"
         machine_env += PG_CONFIG_PREFIX + "tcp_keepalives_count=4" + "\n"
     else:
+        k8s_env.append({
+            CONTAINER_ENV_NAME: PG_CONFIG_PREFIX + "shared_preload_libraries",
+            CONTAINER_ENV_VALUE: "'citus,pgautofailover,pg_stat_statements'"
+        })
         k8s_env.append({
             CONTAINER_ENV_NAME: PG_CONFIG_PREFIX + "log_truncate_on_rotation",
             CONTAINER_ENV_VALUE: "true"
@@ -2512,6 +2518,8 @@ async def create_cluster(
         logger.info("waiting for create_cluster success")
         waiting_cluster_final_status(meta, spec, patch, status, logger)
 
+        update_pgpassfile(meta, spec, patch, status, logger)
+
         # wait a few seconds to prevent the pod not running
         time.sleep(5)
         # cluster running
@@ -3464,6 +3472,52 @@ def update_hbas(
         readwrite_conns.free_conns()
         readonly_conns.free_conns()
 
+def get_pgpassfile(
+    meta: kopf.Meta,
+    spec: kopf.Spec,
+    patch: kopf.Patch,
+    status: kopf.Status,
+    logger: logging.Logger,
+    users_kind: str,
+) -> str:
+    pgpassfile = ""
+    if spec[POSTGRESQL][SPEC_POSTGRESQL_USERS].get(users_kind) != None:
+        users = spec[POSTGRESQL][SPEC_POSTGRESQL_USERS][users_kind]
+        for user in users:
+            pgpassfile += "*:*:*:%s:%s\n" % (user[SPEC_POSTGRESQL_USERS_USER_NAME], user[SPEC_POSTGRESQL_USERS_USER_PASSWORD])
+
+    return pgpassfile
+
+def update_pgpassfile(
+    meta: kopf.Meta,
+    spec: kopf.Spec,
+    patch: kopf.Patch,
+    status: kopf.Status,
+    logger: logging.Logger,
+) -> None:
+    pgpassfile = ""
+    pgpassfile += get_pgpassfile(meta, spec, patch, status, logger, SPEC_POSTGRESQL_USERS_ADMIN)
+    pgpassfile += get_pgpassfile(meta, spec, patch, status, logger, SPEC_POSTGRESQL_USERS_MAINTENANCE)
+    pgpassfile += get_pgpassfile(meta, spec, patch, status, logger, SPEC_POSTGRESQL_USERS_NORMAL)
+
+    if len(pgpassfile) == 0:
+        return
+
+    logger.info(f"update pgpassfile, {pgpassfile}")
+    conns = connections(spec, meta, patch,
+                        get_field(POSTGRESQL, READWRITEINSTANCE), False,
+                        None, logger, None, status, False)
+    readonly_conns = connections(spec, meta, patch,
+                                 get_field(POSTGRESQL, READONLYINSTANCE),
+                                 False, None, logger, None, status, False)
+    for conn in (conns.get_conns() + readonly_conns.get_conns()):
+        cmd = ["echo", "-e", '"' + pgpassfile + '"', ">", PGPASSFILE_PATH]
+        output = exec_command(conn, cmd, logger, interrupt=False)
+        cmd = ["chmod", "0600", PGPASSFILE_PATH]
+        output = exec_command(conn, cmd, logger, interrupt=False)
+    conns.free_conns()
+    readonly_conns.free_conns()
+
 
 def update_users(
     meta: kopf.Meta,
@@ -3480,6 +3534,8 @@ def update_users(
             or FIELD == DIFF_FIELD_POSTGRESQL_USERS_ADMIN \
             or FIELD == DIFF_FIELD_POSTGRESQL_USERS_MAINTENANCE \
             or FIELD == DIFF_FIELD_POSTGRESQL_USERS_NORMAL:
+        update_pgpassfile(meta, spec, patch, status, logger)
+
         conns = connections(spec, meta, patch,
                             get_field(POSTGRESQL, READWRITEINSTANCE), False,
                             None, logger, None, status, False)

source_code/.gitignore

@@ -1,3 +1,5 @@
 pg_auto_failover
 postgresql
 pg_dirtyread
+citus
+postgis

source_code/download.sh

@@ -56,4 +56,18 @@ else
 	echo "download pg_dirtyread success"
 fi
 
+# citus
+if [ -d citus ]
+then
+	echo "citus exists, skip download."
+else
+	git clone https://github.com/citusdata/citus.git
+	if [ $? -ne 0 ]
+	then
+		echo "download citus failed"
+		exit 1
+	fi
+	echo "download citus success"
+fi
+
 exit 0