From 517c64f87e6661366b415df3f2273c76cea428b0 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 28 May 2021 10:51:25 -0700 Subject: [PATCH] Add initial jq-based templating engine --- .gitattributes | 3 + .github/workflows/verify-templating.yml | 22 ++ .gitignore | 1 + 10/alpine/Dockerfile | 13 +- {12 => 10/buster}/Dockerfile | 27 +- 10/{ => buster}/docker-entrypoint.sh | 0 10/{ => stretch}/Dockerfile | 23 +- {11 => 10/stretch}/docker-entrypoint.sh | 0 11/alpine/Dockerfile | 14 +- 11/buster/Dockerfile | 216 ++++++++++++++++ {12 => 11/buster}/docker-entrypoint.sh | 0 11/{ => stretch}/Dockerfile | 23 +- {13 => 11/stretch}/docker-entrypoint.sh | 0 12/alpine/Dockerfile | 13 +- 12/buster/Dockerfile | 216 ++++++++++++++++ 12/buster/docker-entrypoint.sh | 327 ++++++++++++++++++++++++ 13/alpine/Dockerfile | 13 +- 13/{ => buster}/Dockerfile | 24 +- 13/buster/docker-entrypoint.sh | 327 ++++++++++++++++++++++++ 9.6/alpine/Dockerfile | 12 +- 9.6/buster/Dockerfile | 218 ++++++++++++++++ 9.6/{ => buster}/docker-entrypoint.sh | 0 9.6/{ => stretch}/Dockerfile | 23 +- 9.6/stretch/docker-entrypoint.sh | 327 ++++++++++++++++++++++++ Dockerfile-alpine.template | 25 +- Dockerfile-debian.template | 36 +-- apply-templates.sh | 66 +++++ generate-stackbrew-library.sh | 73 +++--- update.sh | 164 +----------- versions.json | 124 +++++++++ versions.sh | 153 +++++++++++ 31 files changed, 2193 insertions(+), 290 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/verify-templating.yml create mode 100644 .gitignore rename {12 => 10/buster}/Dockerfile (95%) rename 10/{ => buster}/docker-entrypoint.sh (100%) rename 10/{ => stretch}/Dockerfile (95%) rename {11 => 10/stretch}/docker-entrypoint.sh (100%) create mode 100644 11/buster/Dockerfile rename {12 => 11/buster}/docker-entrypoint.sh (100%) rename 11/{ => stretch}/Dockerfile (96%) rename {13 => 11/stretch}/docker-entrypoint.sh (100%) create mode 100644 12/buster/Dockerfile create mode 100755 12/buster/docker-entrypoint.sh rename 13/{ => buster}/Dockerfile (95%) create mode 100755 13/buster/docker-entrypoint.sh create mode 100644 9.6/buster/Dockerfile rename 9.6/{ => buster}/docker-entrypoint.sh (100%) rename 9.6/{ => stretch}/Dockerfile (95%) create mode 100755 9.6/stretch/docker-entrypoint.sh create mode 100755 apply-templates.sh create mode 100644 versions.json create mode 100755 versions.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..14a112269e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +/*/**/Dockerfile linguist-generated +/*/**/docker-entrypoint.sh linguist-generated +/Dockerfile*.template linguist-language=Dockerfile diff --git a/.github/workflows/verify-templating.yml b/.github/workflows/verify-templating.yml new file mode 100644 index 0000000000..7e833f1c7d --- /dev/null +++ b/.github/workflows/verify-templating.yml @@ -0,0 +1,22 @@ +name: Verify Templating + +on: + pull_request: + push: + +defaults: + run: + shell: 'bash -Eeuo pipefail -x {0}' + +jobs: + apply-templates: + name: Check For Uncomitted Changes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Apply Templates + run: ./apply-templates.sh + - name: Check Git Status + run: | + status="$(git status --short)" + [ -z "$status" ] diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..d548f66de0 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.jq-template.awk diff --git a/10/alpine/Dockerfile b/10/alpine/Dockerfile index d4ed3a564d..8131b4addd 100644 --- a/10/alpine/Dockerfile +++ b/10/alpine/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.14 # 70 is the standard uid/gid for "postgres" in Alpine @@ -59,6 +64,7 @@ RUN set -eux; \ # tcl-dev \ util-linux-dev \ zlib-dev \ +# https://www.postgresql.org/docs/10/static/release-10.html#id-1.11.6.9.5.13 icu-dev \ ; \ \ @@ -134,7 +140,10 @@ RUN set -eux; \ postgres --version # make the sample config easier to munge (and "correct by default") -RUN sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample +RUN set -eux; \ + cp -v /usr/local/share/postgresql/postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample.orig; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/local/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/12/Dockerfile b/10/buster/Dockerfile similarity index 95% rename from 12/Dockerfile rename to 10/buster/Dockerfile index 1a4dd6f7b4..795143f4c5 100644 --- a/12/Dockerfile +++ b/10/buster/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM debian:buster-slim RUN set -ex; \ @@ -82,8 +87,10 @@ RUN set -ex; \ rm -rf "$GNUPGHOME"; \ apt-key list -ENV PG_MAJOR 12 -ENV PG_VERSION 12.7-1.pgdg100+1 +ENV PG_MAJOR 10 +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + +ENV PG_VERSION 10.17-1.pgdg100+1 RUN set -ex; \ \ @@ -102,15 +109,6 @@ RUN set -ex; \ # let's build binaries from their published source packages echo "deb-src http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ \ - case "$PG_MAJOR" in \ - 9.* | 10 ) ;; \ - *) \ -# https://github.com/docker-library/postgres/issues/484 (clang-6.0 required, only available in stretch-backports) -# TODO remove this once we hit buster+ - echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list.d/pgdg.list; \ - ;; \ - esac; \ - \ tempDir="$(mktemp -d)"; \ cd "$tempDir"; \ \ @@ -162,7 +160,9 @@ RUN set -ex; \ fi; \ \ # some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) - find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' + + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version # make the sample config easier to munge (and "correct by default") RUN set -eux; \ @@ -174,7 +174,6 @@ RUN set -eux; \ RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql -ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin ENV PGDATA /var/lib/postgresql/data # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" diff --git a/10/docker-entrypoint.sh b/10/buster/docker-entrypoint.sh similarity index 100% rename from 10/docker-entrypoint.sh rename to 10/buster/docker-entrypoint.sh diff --git a/10/Dockerfile b/10/stretch/Dockerfile similarity index 95% rename from 10/Dockerfile rename to 10/stretch/Dockerfile index cab271eb09..2b7eb9ca11 100644 --- a/10/Dockerfile +++ b/10/stretch/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM debian:stretch-slim RUN set -ex; \ @@ -83,6 +88,8 @@ RUN set -ex; \ apt-key list ENV PG_MAJOR 10 +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + ENV PG_VERSION 10.17-1.pgdg90+1 RUN set -ex; \ @@ -102,15 +109,6 @@ RUN set -ex; \ # let's build binaries from their published source packages echo "deb-src http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ \ - case "$PG_MAJOR" in \ - 9.* | 10 ) ;; \ - *) \ -# https://github.com/docker-library/postgres/issues/484 (clang-6.0 required, only available in stretch-backports) -# TODO remove this once we hit buster+ - echo 'deb http://deb.debian.org/debian stretch-backports main' >> /etc/apt/sources.list.d/pgdg.list; \ - ;; \ - esac; \ - \ tempDir="$(mktemp -d)"; \ cd "$tempDir"; \ \ @@ -162,7 +160,9 @@ RUN set -ex; \ fi; \ \ # some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) - find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' + + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version # make the sample config easier to munge (and "correct by default") RUN set -eux; \ @@ -174,7 +174,6 @@ RUN set -eux; \ RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql -ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin ENV PGDATA /var/lib/postgresql/data # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" diff --git a/11/docker-entrypoint.sh b/10/stretch/docker-entrypoint.sh similarity index 100% rename from 11/docker-entrypoint.sh rename to 10/stretch/docker-entrypoint.sh diff --git a/11/alpine/Dockerfile b/11/alpine/Dockerfile index 2e66483aa5..acf676c668 100644 --- a/11/alpine/Dockerfile +++ b/11/alpine/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.14 # 70 is the standard uid/gid for "postgres" in Alpine @@ -60,6 +65,7 @@ RUN set -eux; \ # tcl-dev \ util-linux-dev \ zlib-dev \ +# https://www.postgresql.org/docs/10/static/release-10.html#id-1.11.6.9.5.13 icu-dev \ ; \ \ @@ -136,7 +142,10 @@ RUN set -eux; \ postgres --version # make the sample config easier to munge (and "correct by default") -RUN sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample +RUN set -eux; \ + cp -v /usr/local/share/postgresql/postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample.orig; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/local/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql @@ -146,7 +155,6 @@ RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PG VOLUME /var/lib/postgresql/data COPY docker-entrypoint.sh /usr/local/bin/ -RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat ENTRYPOINT ["docker-entrypoint.sh"] # We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL diff --git a/11/buster/Dockerfile b/11/buster/Dockerfile new file mode 100644 index 0000000000..527456c266 --- /dev/null +++ b/11/buster/Dockerfile @@ -0,0 +1,216 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + +FROM debian:buster-slim + +RUN set -ex; \ + if ! command -v gpg > /dev/null; then \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + gnupg \ + dirmngr \ + ; \ + rm -rf /var/lib/apt/lists/*; \ + fi + +# explicitly set user/group IDs +RUN set -eux; \ + groupadd -r postgres --gid=999; \ +# https://salsa.debian.org/postgresql/postgresql-common/blob/997d842ee744687d99a2b2d95c1083a2615c79e8/debian/postgresql-common.postinst#L32-35 + useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \ +# also create the postgres user's home directory with appropriate permissions +# see https://github.com/docker-library/postgres/issues/274 + mkdir -p /var/lib/postgresql; \ + chown -R postgres:postgres /var/lib/postgresql + +# grab gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.12 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default +RUN set -eux; \ + if [ -f /etc/dpkg/dpkg.cfg.d/docker ]; then \ +# if this file exists, we're likely in "debian:xxx-slim", and locales are thus being excluded so we need to remove that exclusion (since we need locales) + grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; \ + sed -ri '/\/usr\/share\/locale/d' /etc/dpkg/dpkg.cfg.d/docker; \ + ! grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; \ + fi; \ + apt-get update; apt-get install -y --no-install-recommends locales; rm -rf /var/lib/apt/lists/*; \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 +ENV LANG en_US.utf8 + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ +# install "nss_wrapper" in case we need to fake "/etc/passwd" and "/etc/group" (especially for OpenShift) +# https://github.com/docker-library/postgres/issues/359 +# https://cwrap.org/nss_wrapper.html + libnss-wrapper \ +# install "xz-utils" for .sql.xz docker-entrypoint-initdb.d files + xz-utils \ + ; \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir /docker-entrypoint-initdb.d + +RUN set -ex; \ +# pub 4096R/ACCC4CF8 2011-10-13 [expires: 2019-07-02] +# Key fingerprint = B97B 0AFC AA1A 47F0 44F2 44A0 7FCC 7D46 ACCC 4CF8 +# uid PostgreSQL Debian Repository + key='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8'; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \ + gpg --batch --export "$key" > /etc/apt/trusted.gpg.d/postgres.gpg; \ + command -v gpgconf > /dev/null && gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + apt-key list + +ENV PG_MAJOR 11 +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + +ENV PG_VERSION 11.12-1.pgdg100+1 + +RUN set -ex; \ + \ +# see note below about "*.pyc" files + export PYTHONDONTWRITEBYTECODE=1; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64 | arm64 | i386 | ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y --no-install-recommends postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y --no-install-recommends \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi; \ + \ +# some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version + +# make the sample config easier to munge (and "correct by default") +RUN set -eux; \ + dpkg-divert --add --rename --divert "/usr/share/postgresql/postgresql.conf.sample.dpkg" "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample"; \ + cp -v /usr/share/postgresql/postgresql.conf.sample.dpkg /usr/share/postgresql/postgresql.conf.sample; \ + ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/"; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/share/postgresql/postgresql.conf.sample + +RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql + +ENV PGDATA /var/lib/postgresql/data +# this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) +RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" +VOLUME /var/lib/postgresql/data + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +# We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL +# calls "Fast Shutdown mode" wherein new connections are disallowed and any +# in-progress transactions are aborted, allowing PostgreSQL to stop cleanly and +# flush tables to disk, which is the best compromise available to avoid data +# corruption. +# +# Users who know their applications do not keep open long-lived idle connections +# may way to use a value of SIGTERM instead, which corresponds to "Smart +# Shutdown mode" in which any existing sessions are allowed to finish and the +# server stops when all sessions are terminated. +# +# See https://www.postgresql.org/docs/12/server-shutdown.html for more details +# about available PostgreSQL server shutdown signals. +# +# See also https://www.postgresql.org/docs/12/server-start.html for further +# justification of this as the default value, namely that the example (and +# shipped) systemd service files use the "Fast Shutdown mode" for service +# termination. +# +STOPSIGNAL SIGINT +# +# An additional setting that is recommended for all users regardless of this +# value is the runtime "--stop-timeout" (or your orchestrator/runtime's +# equivalent) for controlling how long to wait between sending the defined +# STOPSIGNAL and sending SIGKILL (which is likely to cause data corruption). +# +# The default in most runtimes (such as Docker) is 10 seconds, and the +# documentation at https://www.postgresql.org/docs/12/server-start.html notes +# that even 90 seconds may not be long enough in many instances. + +EXPOSE 5432 +CMD ["postgres"] diff --git a/12/docker-entrypoint.sh b/11/buster/docker-entrypoint.sh similarity index 100% rename from 12/docker-entrypoint.sh rename to 11/buster/docker-entrypoint.sh diff --git a/11/Dockerfile b/11/stretch/Dockerfile similarity index 96% rename from 11/Dockerfile rename to 11/stretch/Dockerfile index 340047fbb8..a3f46109c1 100644 --- a/11/Dockerfile +++ b/11/stretch/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM debian:stretch-slim RUN set -ex; \ @@ -83,6 +88,8 @@ RUN set -ex; \ apt-key list ENV PG_MAJOR 11 +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + ENV PG_VERSION 11.12-1.pgdg90+1 RUN set -ex; \ @@ -102,14 +109,8 @@ RUN set -ex; \ # let's build binaries from their published source packages echo "deb-src http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ \ - case "$PG_MAJOR" in \ - 9.* | 10 ) ;; \ - *) \ # https://github.com/docker-library/postgres/issues/484 (clang-6.0 required, only available in stretch-backports) -# TODO remove this once we hit buster+ - echo 'deb http://deb.debian.org/debian stretch-backports main' >> /etc/apt/sources.list.d/pgdg.list; \ - ;; \ - esac; \ + echo 'deb http://deb.debian.org/debian stretch-backports main' >> /etc/apt/sources.list.d/pgdg.list; \ \ tempDir="$(mktemp -d)"; \ cd "$tempDir"; \ @@ -162,7 +163,9 @@ RUN set -ex; \ fi; \ \ # some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) - find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' + + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version # make the sample config easier to munge (and "correct by default") RUN set -eux; \ @@ -174,14 +177,12 @@ RUN set -eux; \ RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql -ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin ENV PGDATA /var/lib/postgresql/data # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" VOLUME /var/lib/postgresql/data COPY docker-entrypoint.sh /usr/local/bin/ -RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat ENTRYPOINT ["docker-entrypoint.sh"] # We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL diff --git a/13/docker-entrypoint.sh b/11/stretch/docker-entrypoint.sh similarity index 100% rename from 13/docker-entrypoint.sh rename to 11/stretch/docker-entrypoint.sh diff --git a/12/alpine/Dockerfile b/12/alpine/Dockerfile index 58a0804e24..21f6d42023 100644 --- a/12/alpine/Dockerfile +++ b/12/alpine/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.14 # 70 is the standard uid/gid for "postgres" in Alpine @@ -60,6 +65,7 @@ RUN set -eux; \ # tcl-dev \ util-linux-dev \ zlib-dev \ +# https://www.postgresql.org/docs/10/static/release-10.html#id-1.11.6.9.5.13 icu-dev \ ; \ \ @@ -136,7 +142,10 @@ RUN set -eux; \ postgres --version # make the sample config easier to munge (and "correct by default") -RUN sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample +RUN set -eux; \ + cp -v /usr/local/share/postgresql/postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample.orig; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/local/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/12/buster/Dockerfile b/12/buster/Dockerfile new file mode 100644 index 0000000000..6dafa8a094 --- /dev/null +++ b/12/buster/Dockerfile @@ -0,0 +1,216 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + +FROM debian:buster-slim + +RUN set -ex; \ + if ! command -v gpg > /dev/null; then \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + gnupg \ + dirmngr \ + ; \ + rm -rf /var/lib/apt/lists/*; \ + fi + +# explicitly set user/group IDs +RUN set -eux; \ + groupadd -r postgres --gid=999; \ +# https://salsa.debian.org/postgresql/postgresql-common/blob/997d842ee744687d99a2b2d95c1083a2615c79e8/debian/postgresql-common.postinst#L32-35 + useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \ +# also create the postgres user's home directory with appropriate permissions +# see https://github.com/docker-library/postgres/issues/274 + mkdir -p /var/lib/postgresql; \ + chown -R postgres:postgres /var/lib/postgresql + +# grab gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.12 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default +RUN set -eux; \ + if [ -f /etc/dpkg/dpkg.cfg.d/docker ]; then \ +# if this file exists, we're likely in "debian:xxx-slim", and locales are thus being excluded so we need to remove that exclusion (since we need locales) + grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; \ + sed -ri '/\/usr\/share\/locale/d' /etc/dpkg/dpkg.cfg.d/docker; \ + ! grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; \ + fi; \ + apt-get update; apt-get install -y --no-install-recommends locales; rm -rf /var/lib/apt/lists/*; \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 +ENV LANG en_US.utf8 + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ +# install "nss_wrapper" in case we need to fake "/etc/passwd" and "/etc/group" (especially for OpenShift) +# https://github.com/docker-library/postgres/issues/359 +# https://cwrap.org/nss_wrapper.html + libnss-wrapper \ +# install "xz-utils" for .sql.xz docker-entrypoint-initdb.d files + xz-utils \ + ; \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir /docker-entrypoint-initdb.d + +RUN set -ex; \ +# pub 4096R/ACCC4CF8 2011-10-13 [expires: 2019-07-02] +# Key fingerprint = B97B 0AFC AA1A 47F0 44F2 44A0 7FCC 7D46 ACCC 4CF8 +# uid PostgreSQL Debian Repository + key='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8'; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \ + gpg --batch --export "$key" > /etc/apt/trusted.gpg.d/postgres.gpg; \ + command -v gpgconf > /dev/null && gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + apt-key list + +ENV PG_MAJOR 12 +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + +ENV PG_VERSION 12.7-1.pgdg100+1 + +RUN set -ex; \ + \ +# see note below about "*.pyc" files + export PYTHONDONTWRITEBYTECODE=1; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64 | arm64 | i386 | ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y --no-install-recommends postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y --no-install-recommends \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi; \ + \ +# some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version + +# make the sample config easier to munge (and "correct by default") +RUN set -eux; \ + dpkg-divert --add --rename --divert "/usr/share/postgresql/postgresql.conf.sample.dpkg" "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample"; \ + cp -v /usr/share/postgresql/postgresql.conf.sample.dpkg /usr/share/postgresql/postgresql.conf.sample; \ + ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/"; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/share/postgresql/postgresql.conf.sample + +RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql + +ENV PGDATA /var/lib/postgresql/data +# this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) +RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" +VOLUME /var/lib/postgresql/data + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +# We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL +# calls "Fast Shutdown mode" wherein new connections are disallowed and any +# in-progress transactions are aborted, allowing PostgreSQL to stop cleanly and +# flush tables to disk, which is the best compromise available to avoid data +# corruption. +# +# Users who know their applications do not keep open long-lived idle connections +# may way to use a value of SIGTERM instead, which corresponds to "Smart +# Shutdown mode" in which any existing sessions are allowed to finish and the +# server stops when all sessions are terminated. +# +# See https://www.postgresql.org/docs/12/server-shutdown.html for more details +# about available PostgreSQL server shutdown signals. +# +# See also https://www.postgresql.org/docs/12/server-start.html for further +# justification of this as the default value, namely that the example (and +# shipped) systemd service files use the "Fast Shutdown mode" for service +# termination. +# +STOPSIGNAL SIGINT +# +# An additional setting that is recommended for all users regardless of this +# value is the runtime "--stop-timeout" (or your orchestrator/runtime's +# equivalent) for controlling how long to wait between sending the defined +# STOPSIGNAL and sending SIGKILL (which is likely to cause data corruption). +# +# The default in most runtimes (such as Docker) is 10 seconds, and the +# documentation at https://www.postgresql.org/docs/12/server-start.html notes +# that even 90 seconds may not be long enough in many instances. + +EXPOSE 5432 +CMD ["postgres"] diff --git a/12/buster/docker-entrypoint.sh b/12/buster/docker-entrypoint.sh new file mode 100755 index 0000000000..eeeac649d0 --- /dev/null +++ b/12/buster/docker-entrypoint.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +set -Eeo pipefail +# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user +docker_create_db_directories() { + local user; user="$(id -u)" + + mkdir -p "$PGDATA" + # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) + chmod 700 "$PGDATA" || : + + # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 + mkdir -p /var/run/postgresql || : + chmod 775 /var/run/postgresql || : + + # Create the transaction log directory before initdb is run so the directory is owned by the correct user + if [ -n "$POSTGRES_INITDB_WALDIR" ]; then + mkdir -p "$POSTGRES_INITDB_WALDIR" + if [ "$user" = '0' ]; then + find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + + fi + chmod 700 "$POSTGRES_INITDB_WALDIR" + fi + + # allow the container to be started with `--user` + if [ "$user" = '0' ]; then + find "$PGDATA" \! -user postgres -exec chown postgres '{}' + + find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + + fi +} + +# initialize empty PGDATA directory with new database via 'initdb' +# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function +# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames +# this is also where the database user is created, specified by `POSTGRES_USER` env +docker_init_database_dir() { + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + if ! getent passwd "$(id -u)" &> /dev/null && [ -e /usr/lib/libnss_wrapper.so ]; then + export LD_PRELOAD='/usr/lib/libnss_wrapper.so' + export NSS_WRAPPER_PASSWD="$(mktemp)" + export NSS_WRAPPER_GROUP="$(mktemp)" + echo "postgres:x:$(id -u):$(id -g):PostgreSQL:$PGDATA:/bin/false" > "$NSS_WRAPPER_PASSWD" + echo "postgres:x:$(id -g):" > "$NSS_WRAPPER_GROUP" + fi + + if [ -n "$POSTGRES_INITDB_WALDIR" ]; then + set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" + fi + + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(echo "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' + + # unset/cleanup "nss_wrapper" bits + if [ "${LD_PRELOAD:-}" = '/usr/lib/libnss_wrapper.so' ]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi +} + +# print large warning if POSTGRES_PASSWORD is long +# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' +# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' +# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] +docker_verify_minimum_env() { + # check password first so we can output the warning before postgres + # messes it up + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOE' + Error: Database is uninitialized and superuser password is not specified. + You must specify POSTGRES_PASSWORD to a non-empty value for the + superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". + + You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all + connections without a password. This is *not* recommended. + + See PostgreSQL documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + EOE + exit 1 + fi + if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then + cat >&2 <<-'EOWARN' + ******************************************************************************** + WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow + anyone with access to the Postgres port to access your database without + a password, even if POSTGRES_PASSWORD is set. See PostgreSQL + documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + In Docker's default configuration, this is effectively any other + container on the same system. + + It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace + it with "-e POSTGRES_PASSWORD=password" instead to set a password in + "docker run". + ******************************************************************************** + EOWARN + fi +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions and permissions +docker_process_init_files() { + # psql here for backwards compatibility "${psql[@]}" + psql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; docker_process_sql -f "$f"; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) echo "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done +} + +# Execute sql script, passed via stdin (or -f flag of pqsl) +# usage: docker_process_sql [psql-cli-args] +# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' +# ie: docker_process_sql -f my-file.sql +# ie: docker_process_sql > "$PGDATA/pg_hba.conf" +} + +# start socket-only postgresql server for setting up or running scripts +# all arguments will be passed along as arguments to `postgres` (via pg_ctl) +docker_temp_server_start() { + if [ "$1" = 'postgres' ]; then + shift + fi + + # internal start of server in order to allow setup using psql client + # does not listen on external TCP/IP and waits until start finishes + set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "$(printf '%q ' "$@")" \ + -w start +} + +# stop postgresql server after done setting up user and running scripts +docker_temp_server_stop() { + PGUSER="${PGUSER:-postgres}" \ + pg_ctl -D "$PGDATA" -m fast -w stop +} + +# check arguments for an option that would cause postgres to stop +# return true if there is one +_pg_want_help() { + local arg + for arg; do + case "$arg" in + # postgres --help | grep 'then exit' + # leaving out -C on purpose since it always fails and is unhelpful: + # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory + -'?'|--help|--describe-config|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +_main() { + # if first arg looks like a flag, assume we want to run postgres server + if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" + fi + + if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then + docker_setup_env + # setup data directories and permissions (when run as root) + docker_create_db_directories + if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec gosu postgres "$BASH_SOURCE" "$@" + fi + + # only run initialization on an empty data directory + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD + + echo + echo 'PostgreSQL init process complete; ready for start up.' + echo + else + echo + echo 'PostgreSQL Database directory appears to contain a database; Skipping initialization' + echo + fi + fi + + exec "$@" +} + +if ! _is_sourced; then + _main "$@" +fi diff --git a/13/alpine/Dockerfile b/13/alpine/Dockerfile index 21e8257141..2148b0674f 100644 --- a/13/alpine/Dockerfile +++ b/13/alpine/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.14 # 70 is the standard uid/gid for "postgres" in Alpine @@ -60,6 +65,7 @@ RUN set -eux; \ # tcl-dev \ util-linux-dev \ zlib-dev \ +# https://www.postgresql.org/docs/10/static/release-10.html#id-1.11.6.9.5.13 icu-dev \ ; \ \ @@ -136,7 +142,10 @@ RUN set -eux; \ postgres --version # make the sample config easier to munge (and "correct by default") -RUN sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample +RUN set -eux; \ + cp -v /usr/local/share/postgresql/postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample.orig; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/local/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/13/Dockerfile b/13/buster/Dockerfile similarity index 95% rename from 13/Dockerfile rename to 13/buster/Dockerfile index 46f1c2a2d0..6ce45fa7db 100644 --- a/13/Dockerfile +++ b/13/buster/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM debian:buster-slim RUN set -ex; \ @@ -83,6 +88,8 @@ RUN set -ex; \ apt-key list ENV PG_MAJOR 13 +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + ENV PG_VERSION 13.3-1.pgdg100+1 RUN set -ex; \ @@ -102,15 +109,6 @@ RUN set -ex; \ # let's build binaries from their published source packages echo "deb-src http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ \ - case "$PG_MAJOR" in \ - 9.* | 10 ) ;; \ - *) \ -# https://github.com/docker-library/postgres/issues/484 (clang-6.0 required, only available in stretch-backports) -# TODO remove this once we hit buster+ - echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list.d/pgdg.list; \ - ;; \ - esac; \ - \ tempDir="$(mktemp -d)"; \ cd "$tempDir"; \ \ @@ -164,7 +162,9 @@ RUN set -ex; \ fi; \ \ # some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) - find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' + + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version # make the sample config easier to munge (and "correct by default") RUN set -eux; \ @@ -176,14 +176,12 @@ RUN set -eux; \ RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql -ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin ENV PGDATA /var/lib/postgresql/data # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" VOLUME /var/lib/postgresql/data COPY docker-entrypoint.sh /usr/local/bin/ -RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat ENTRYPOINT ["docker-entrypoint.sh"] # We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL diff --git a/13/buster/docker-entrypoint.sh b/13/buster/docker-entrypoint.sh new file mode 100755 index 0000000000..eeeac649d0 --- /dev/null +++ b/13/buster/docker-entrypoint.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +set -Eeo pipefail +# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user +docker_create_db_directories() { + local user; user="$(id -u)" + + mkdir -p "$PGDATA" + # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) + chmod 700 "$PGDATA" || : + + # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 + mkdir -p /var/run/postgresql || : + chmod 775 /var/run/postgresql || : + + # Create the transaction log directory before initdb is run so the directory is owned by the correct user + if [ -n "$POSTGRES_INITDB_WALDIR" ]; then + mkdir -p "$POSTGRES_INITDB_WALDIR" + if [ "$user" = '0' ]; then + find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + + fi + chmod 700 "$POSTGRES_INITDB_WALDIR" + fi + + # allow the container to be started with `--user` + if [ "$user" = '0' ]; then + find "$PGDATA" \! -user postgres -exec chown postgres '{}' + + find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + + fi +} + +# initialize empty PGDATA directory with new database via 'initdb' +# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function +# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames +# this is also where the database user is created, specified by `POSTGRES_USER` env +docker_init_database_dir() { + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + if ! getent passwd "$(id -u)" &> /dev/null && [ -e /usr/lib/libnss_wrapper.so ]; then + export LD_PRELOAD='/usr/lib/libnss_wrapper.so' + export NSS_WRAPPER_PASSWD="$(mktemp)" + export NSS_WRAPPER_GROUP="$(mktemp)" + echo "postgres:x:$(id -u):$(id -g):PostgreSQL:$PGDATA:/bin/false" > "$NSS_WRAPPER_PASSWD" + echo "postgres:x:$(id -g):" > "$NSS_WRAPPER_GROUP" + fi + + if [ -n "$POSTGRES_INITDB_WALDIR" ]; then + set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" + fi + + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(echo "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' + + # unset/cleanup "nss_wrapper" bits + if [ "${LD_PRELOAD:-}" = '/usr/lib/libnss_wrapper.so' ]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi +} + +# print large warning if POSTGRES_PASSWORD is long +# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' +# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' +# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] +docker_verify_minimum_env() { + # check password first so we can output the warning before postgres + # messes it up + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOE' + Error: Database is uninitialized and superuser password is not specified. + You must specify POSTGRES_PASSWORD to a non-empty value for the + superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". + + You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all + connections without a password. This is *not* recommended. + + See PostgreSQL documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + EOE + exit 1 + fi + if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then + cat >&2 <<-'EOWARN' + ******************************************************************************** + WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow + anyone with access to the Postgres port to access your database without + a password, even if POSTGRES_PASSWORD is set. See PostgreSQL + documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + In Docker's default configuration, this is effectively any other + container on the same system. + + It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace + it with "-e POSTGRES_PASSWORD=password" instead to set a password in + "docker run". + ******************************************************************************** + EOWARN + fi +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions and permissions +docker_process_init_files() { + # psql here for backwards compatibility "${psql[@]}" + psql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; docker_process_sql -f "$f"; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) echo "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done +} + +# Execute sql script, passed via stdin (or -f flag of pqsl) +# usage: docker_process_sql [psql-cli-args] +# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' +# ie: docker_process_sql -f my-file.sql +# ie: docker_process_sql > "$PGDATA/pg_hba.conf" +} + +# start socket-only postgresql server for setting up or running scripts +# all arguments will be passed along as arguments to `postgres` (via pg_ctl) +docker_temp_server_start() { + if [ "$1" = 'postgres' ]; then + shift + fi + + # internal start of server in order to allow setup using psql client + # does not listen on external TCP/IP and waits until start finishes + set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "$(printf '%q ' "$@")" \ + -w start +} + +# stop postgresql server after done setting up user and running scripts +docker_temp_server_stop() { + PGUSER="${PGUSER:-postgres}" \ + pg_ctl -D "$PGDATA" -m fast -w stop +} + +# check arguments for an option that would cause postgres to stop +# return true if there is one +_pg_want_help() { + local arg + for arg; do + case "$arg" in + # postgres --help | grep 'then exit' + # leaving out -C on purpose since it always fails and is unhelpful: + # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory + -'?'|--help|--describe-config|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +_main() { + # if first arg looks like a flag, assume we want to run postgres server + if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" + fi + + if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then + docker_setup_env + # setup data directories and permissions (when run as root) + docker_create_db_directories + if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec gosu postgres "$BASH_SOURCE" "$@" + fi + + # only run initialization on an empty data directory + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD + + echo + echo 'PostgreSQL init process complete; ready for start up.' + echo + else + echo + echo 'PostgreSQL Database directory appears to contain a database; Skipping initialization' + echo + fi + fi + + exec "$@" +} + +if ! _is_sourced; then + _main "$@" +fi diff --git a/9.6/alpine/Dockerfile b/9.6/alpine/Dockerfile index 2abf342b1b..187747a0fa 100644 --- a/9.6/alpine/Dockerfile +++ b/9.6/alpine/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.14 # 70 is the standard uid/gid for "postgres" in Alpine @@ -132,7 +137,10 @@ RUN set -eux; \ postgres --version # make the sample config easier to munge (and "correct by default") -RUN sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample +RUN set -eux; \ + cp -v /usr/local/share/postgresql/postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample.orig; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/local/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/9.6/buster/Dockerfile b/9.6/buster/Dockerfile new file mode 100644 index 0000000000..5c7a42fe61 --- /dev/null +++ b/9.6/buster/Dockerfile @@ -0,0 +1,218 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + +FROM debian:buster-slim + +RUN set -ex; \ + if ! command -v gpg > /dev/null; then \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + gnupg \ + dirmngr \ + ; \ + rm -rf /var/lib/apt/lists/*; \ + fi + +# explicitly set user/group IDs +RUN set -eux; \ + groupadd -r postgres --gid=999; \ +# https://salsa.debian.org/postgresql/postgresql-common/blob/997d842ee744687d99a2b2d95c1083a2615c79e8/debian/postgresql-common.postinst#L32-35 + useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \ +# also create the postgres user's home directory with appropriate permissions +# see https://github.com/docker-library/postgres/issues/274 + mkdir -p /var/lib/postgresql; \ + chown -R postgres:postgres /var/lib/postgresql + +# grab gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.12 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default +RUN set -eux; \ + if [ -f /etc/dpkg/dpkg.cfg.d/docker ]; then \ +# if this file exists, we're likely in "debian:xxx-slim", and locales are thus being excluded so we need to remove that exclusion (since we need locales) + grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; \ + sed -ri '/\/usr\/share\/locale/d' /etc/dpkg/dpkg.cfg.d/docker; \ + ! grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; \ + fi; \ + apt-get update; apt-get install -y --no-install-recommends locales; rm -rf /var/lib/apt/lists/*; \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 +ENV LANG en_US.utf8 + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ +# install "nss_wrapper" in case we need to fake "/etc/passwd" and "/etc/group" (especially for OpenShift) +# https://github.com/docker-library/postgres/issues/359 +# https://cwrap.org/nss_wrapper.html + libnss-wrapper \ +# install "xz-utils" for .sql.xz docker-entrypoint-initdb.d files + xz-utils \ + ; \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir /docker-entrypoint-initdb.d + +RUN set -ex; \ +# pub 4096R/ACCC4CF8 2011-10-13 [expires: 2019-07-02] +# Key fingerprint = B97B 0AFC AA1A 47F0 44F2 44A0 7FCC 7D46 ACCC 4CF8 +# uid PostgreSQL Debian Repository + key='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8'; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \ + gpg --batch --export "$key" > /etc/apt/trusted.gpg.d/postgres.gpg; \ + command -v gpgconf > /dev/null && gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + apt-key list + +ENV PG_MAJOR 9.6 +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + +ENV PG_VERSION 9.6.22-1.pgdg100+1 + +RUN set -ex; \ + \ +# see note below about "*.pyc" files + export PYTHONDONTWRITEBYTECODE=1; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64 | arm64 | i386 | ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y --no-install-recommends postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y --no-install-recommends \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + "postgresql-contrib-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi; \ + \ +# some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version + +# make the sample config easier to munge (and "correct by default") +RUN set -eux; \ + dpkg-divert --add --rename --divert "/usr/share/postgresql/postgresql.conf.sample.dpkg" "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample"; \ + cp -v /usr/share/postgresql/postgresql.conf.sample.dpkg /usr/share/postgresql/postgresql.conf.sample; \ + ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/"; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/share/postgresql/postgresql.conf.sample + +RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql + +ENV PGDATA /var/lib/postgresql/data +# this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) +RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" +VOLUME /var/lib/postgresql/data + +COPY docker-entrypoint.sh /usr/local/bin/ +RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat +ENTRYPOINT ["docker-entrypoint.sh"] + +# We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL +# calls "Fast Shutdown mode" wherein new connections are disallowed and any +# in-progress transactions are aborted, allowing PostgreSQL to stop cleanly and +# flush tables to disk, which is the best compromise available to avoid data +# corruption. +# +# Users who know their applications do not keep open long-lived idle connections +# may way to use a value of SIGTERM instead, which corresponds to "Smart +# Shutdown mode" in which any existing sessions are allowed to finish and the +# server stops when all sessions are terminated. +# +# See https://www.postgresql.org/docs/12/server-shutdown.html for more details +# about available PostgreSQL server shutdown signals. +# +# See also https://www.postgresql.org/docs/12/server-start.html for further +# justification of this as the default value, namely that the example (and +# shipped) systemd service files use the "Fast Shutdown mode" for service +# termination. +# +STOPSIGNAL SIGINT +# +# An additional setting that is recommended for all users regardless of this +# value is the runtime "--stop-timeout" (or your orchestrator/runtime's +# equivalent) for controlling how long to wait between sending the defined +# STOPSIGNAL and sending SIGKILL (which is likely to cause data corruption). +# +# The default in most runtimes (such as Docker) is 10 seconds, and the +# documentation at https://www.postgresql.org/docs/12/server-start.html notes +# that even 90 seconds may not be long enough in many instances. + +EXPOSE 5432 +CMD ["postgres"] diff --git a/9.6/docker-entrypoint.sh b/9.6/buster/docker-entrypoint.sh similarity index 100% rename from 9.6/docker-entrypoint.sh rename to 9.6/buster/docker-entrypoint.sh diff --git a/9.6/Dockerfile b/9.6/stretch/Dockerfile similarity index 95% rename from 9.6/Dockerfile rename to 9.6/stretch/Dockerfile index fa1ce0f268..ddd03bbf6e 100644 --- a/9.6/Dockerfile +++ b/9.6/stretch/Dockerfile @@ -1,4 +1,9 @@ -# vim:set ft=dockerfile: +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM debian:stretch-slim RUN set -ex; \ @@ -83,6 +88,8 @@ RUN set -ex; \ apt-key list ENV PG_MAJOR 9.6 +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + ENV PG_VERSION 9.6.22-1.pgdg90+1 RUN set -ex; \ @@ -102,15 +109,6 @@ RUN set -ex; \ # let's build binaries from their published source packages echo "deb-src http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ \ - case "$PG_MAJOR" in \ - 9.* | 10 ) ;; \ - *) \ -# https://github.com/docker-library/postgres/issues/484 (clang-6.0 required, only available in stretch-backports) -# TODO remove this once we hit buster+ - echo 'deb http://deb.debian.org/debian stretch-backports main' >> /etc/apt/sources.list.d/pgdg.list; \ - ;; \ - esac; \ - \ tempDir="$(mktemp -d)"; \ cd "$tempDir"; \ \ @@ -163,7 +161,9 @@ RUN set -ex; \ fi; \ \ # some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) - find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' + + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version # make the sample config easier to munge (and "correct by default") RUN set -eux; \ @@ -175,7 +175,6 @@ RUN set -eux; \ RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql -ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin ENV PGDATA /var/lib/postgresql/data # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" diff --git a/9.6/stretch/docker-entrypoint.sh b/9.6/stretch/docker-entrypoint.sh new file mode 100755 index 0000000000..8c69d50220 --- /dev/null +++ b/9.6/stretch/docker-entrypoint.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +set -Eeo pipefail +# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user +docker_create_db_directories() { + local user; user="$(id -u)" + + mkdir -p "$PGDATA" + # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) + chmod 700 "$PGDATA" || : + + # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 + mkdir -p /var/run/postgresql || : + chmod 775 /var/run/postgresql || : + + # Create the transaction log directory before initdb is run so the directory is owned by the correct user + if [ -n "$POSTGRES_INITDB_XLOGDIR" ]; then + mkdir -p "$POSTGRES_INITDB_XLOGDIR" + if [ "$user" = '0' ]; then + find "$POSTGRES_INITDB_XLOGDIR" \! -user postgres -exec chown postgres '{}' + + fi + chmod 700 "$POSTGRES_INITDB_XLOGDIR" + fi + + # allow the container to be started with `--user` + if [ "$user" = '0' ]; then + find "$PGDATA" \! -user postgres -exec chown postgres '{}' + + find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + + fi +} + +# initialize empty PGDATA directory with new database via 'initdb' +# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function +# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames +# this is also where the database user is created, specified by `POSTGRES_USER` env +docker_init_database_dir() { + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + if ! getent passwd "$(id -u)" &> /dev/null && [ -e /usr/lib/libnss_wrapper.so ]; then + export LD_PRELOAD='/usr/lib/libnss_wrapper.so' + export NSS_WRAPPER_PASSWD="$(mktemp)" + export NSS_WRAPPER_GROUP="$(mktemp)" + echo "postgres:x:$(id -u):$(id -g):PostgreSQL:$PGDATA:/bin/false" > "$NSS_WRAPPER_PASSWD" + echo "postgres:x:$(id -g):" > "$NSS_WRAPPER_GROUP" + fi + + if [ -n "$POSTGRES_INITDB_XLOGDIR" ]; then + set -- --xlogdir "$POSTGRES_INITDB_XLOGDIR" "$@" + fi + + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(echo "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' + + # unset/cleanup "nss_wrapper" bits + if [ "${LD_PRELOAD:-}" = '/usr/lib/libnss_wrapper.so' ]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi +} + +# print large warning if POSTGRES_PASSWORD is long +# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' +# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' +# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] +docker_verify_minimum_env() { + # check password first so we can output the warning before postgres + # messes it up + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOE' + Error: Database is uninitialized and superuser password is not specified. + You must specify POSTGRES_PASSWORD to a non-empty value for the + superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". + + You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all + connections without a password. This is *not* recommended. + + See PostgreSQL documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + EOE + exit 1 + fi + if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then + cat >&2 <<-'EOWARN' + ******************************************************************************** + WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow + anyone with access to the Postgres port to access your database without + a password, even if POSTGRES_PASSWORD is set. See PostgreSQL + documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + In Docker's default configuration, this is effectively any other + container on the same system. + + It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace + it with "-e POSTGRES_PASSWORD=password" instead to set a password in + "docker run". + ******************************************************************************** + EOWARN + fi +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions and permissions +docker_process_init_files() { + # psql here for backwards compatibility "${psql[@]}" + psql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; docker_process_sql -f "$f"; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) echo "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done +} + +# Execute sql script, passed via stdin (or -f flag of pqsl) +# usage: docker_process_sql [psql-cli-args] +# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' +# ie: docker_process_sql -f my-file.sql +# ie: docker_process_sql > "$PGDATA/pg_hba.conf" +} + +# start socket-only postgresql server for setting up or running scripts +# all arguments will be passed along as arguments to `postgres` (via pg_ctl) +docker_temp_server_start() { + if [ "$1" = 'postgres' ]; then + shift + fi + + # internal start of server in order to allow setup using psql client + # does not listen on external TCP/IP and waits until start finishes + set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "$(printf '%q ' "$@")" \ + -w start +} + +# stop postgresql server after done setting up user and running scripts +docker_temp_server_stop() { + PGUSER="${PGUSER:-postgres}" \ + pg_ctl -D "$PGDATA" -m fast -w stop +} + +# check arguments for an option that would cause postgres to stop +# return true if there is one +_pg_want_help() { + local arg + for arg; do + case "$arg" in + # postgres --help | grep 'then exit' + # leaving out -C on purpose since it always fails and is unhelpful: + # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory + -'?'|--help|--describe-config|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +_main() { + # if first arg looks like a flag, assume we want to run postgres server + if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" + fi + + if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then + docker_setup_env + # setup data directories and permissions (when run as root) + docker_create_db_directories + if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec gosu postgres "$BASH_SOURCE" "$@" + fi + + # only run initialization on an empty data directory + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD + + echo + echo 'PostgreSQL init process complete; ready for start up.' + echo + else + echo + echo 'PostgreSQL Database directory appears to contain a database; Skipping initialization' + echo + fi + fi + + exec "$@" +} + +if ! _is_sourced; then + _main "$@" +fi diff --git a/Dockerfile-alpine.template b/Dockerfile-alpine.template index 221cef7989..2a4148219c 100644 --- a/Dockerfile-alpine.template +++ b/Dockerfile-alpine.template @@ -1,5 +1,4 @@ -# vim:set ft=dockerfile: -FROM alpine:%%ALPINE-VERSION%% +FROM alpine:{{ .alpine }} # 70 is the standard uid/gid for "postgres" in Alpine # https://git.alpinelinux.org/aports/tree/main/postgresql/postgresql.pre-install?h=3.12-stable @@ -17,9 +16,9 @@ ENV LANG en_US.utf8 RUN mkdir /docker-entrypoint-initdb.d -ENV PG_MAJOR %%PG_MAJOR%% -ENV PG_VERSION %%PG_VERSION%% -ENV PG_SHA256 %%PG_SHA256%% +ENV PG_MAJOR {{ env.version }} +ENV PG_VERSION {{ .version }} +ENV PG_SHA256 {{ .sha256 }} RUN set -eux; \ \ @@ -46,7 +45,9 @@ RUN set -eux; \ libxml2-dev \ libxslt-dev \ linux-headers \ +{{ if .major >= 11 then ( -}} llvm11-dev clang g++ \ +{{ ) else "" end -}} make \ # openldap-dev \ openssl-dev \ @@ -60,7 +61,10 @@ RUN set -eux; \ # tcl-dev \ util-linux-dev \ zlib-dev \ +{{ if .major >= 10 then ( -}} +# https://www.postgresql.org/docs/10/static/release-10.html#id-1.11.6.9.5.13 icu-dev \ +{{ ) else "" end -}} ; \ \ cd /usr/src/postgresql; \ @@ -104,8 +108,12 @@ RUN set -eux; \ --with-openssl \ --with-libxml \ --with-libxslt \ +{{ if .major >= 10 then ( -}} --with-icu \ +{{ ) else "" end -}} +{{ if .major >= 11 then ( -}} --with-llvm \ +{{ ) else "" end -}} ; \ make -j "$(nproc)" world; \ make install-world; \ @@ -136,7 +144,10 @@ RUN set -eux; \ postgres --version # make the sample config easier to munge (and "correct by default") -RUN sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample +RUN set -eux; \ + cp -v /usr/local/share/postgresql/postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample.orig; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/local/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql @@ -146,7 +157,9 @@ RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PG VOLUME /var/lib/postgresql/data COPY docker-entrypoint.sh /usr/local/bin/ +{{ if .major >= 11 then "" else ( -}} RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat +{{ ) end -}} ENTRYPOINT ["docker-entrypoint.sh"] # We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL diff --git a/Dockerfile-debian.template b/Dockerfile-debian.template index 876229be59..95eb829801 100644 --- a/Dockerfile-debian.template +++ b/Dockerfile-debian.template @@ -1,5 +1,4 @@ -# vim:set ft=dockerfile: -FROM debian:%%DEBIAN_TAG%% +FROM debian:{{ env.variant }}-slim RUN set -ex; \ if ! command -v gpg > /dev/null; then \ @@ -82,8 +81,10 @@ RUN set -ex; \ rm -rf "$GNUPGHOME"; \ apt-key list -ENV PG_MAJOR %%PG_MAJOR%% -ENV PG_VERSION %%PG_VERSION%% +ENV PG_MAJOR {{ env.version }} +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + +ENV PG_VERSION {{ .[env.variant].version }} RUN set -ex; \ \ @@ -92,25 +93,21 @@ RUN set -ex; \ \ dpkgArch="$(dpkg --print-architecture)"; \ case "$dpkgArch" in \ - %%ARCH_LIST%%) \ + {{ .[env.variant].arches | join(" | ") }}) \ # arches officialy built by upstream - echo "deb http://apt.postgresql.org/pub/repos/apt/ %%DEBIAN_SUITE%%-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + echo "deb http://apt.postgresql.org/pub/repos/apt/ {{ env.variant }}-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ apt-get update; \ ;; \ *) \ # we're on an architecture upstream doesn't officially build for # let's build binaries from their published source packages - echo "deb-src http://apt.postgresql.org/pub/repos/apt/ %%DEBIAN_SUITE%%-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ {{ env.variant }}-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ \ - case "$PG_MAJOR" in \ - 9.* | 10 ) ;; \ - *) \ +{{ if env.variant == "stretch" and .major >= 11 then ( -}} # https://github.com/docker-library/postgres/issues/484 (clang-6.0 required, only available in stretch-backports) -# TODO remove this once we hit buster+ - echo 'deb http://deb.debian.org/debian %%DEBIAN_SUITE%%-backports main' >> /etc/apt/sources.list.d/pgdg.list; \ - ;; \ - esac; \ + echo 'deb http://deb.debian.org/debian {{ env.variant }}-backports main' >> /etc/apt/sources.list.d/pgdg.list; \ \ +{{ ) else "" end -}} tempDir="$(mktemp -d)"; \ cd "$tempDir"; \ \ @@ -118,8 +115,10 @@ RUN set -ex; \ \ # build .deb files from upstream's source packages (which are verified by apt-get) apt-get update; \ +{{ if .major == 13 then ( -}} # we need DEBIAN_FRONTEND on postgresql-13 for slapd ("Please enter the password for the admin entry in your LDAP directory."); see https://bugs.debian.org/929417 DEBIAN_FRONTEND=noninteractive \ +{{ ) else "" end -}} apt-get build-dep -y \ postgresql-common pgdg-keyring \ "postgresql-$PG_MAJOR=$PG_VERSION" \ @@ -153,7 +152,9 @@ RUN set -ex; \ sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ apt-get install -y --no-install-recommends \ "postgresql-$PG_MAJOR=$PG_VERSION" \ +{{ if .major == 9 then ( -}} "postgresql-contrib-$PG_MAJOR=$PG_VERSION" \ +{{ ) else "" end -}} ; \ \ rm -rf /var/lib/apt/lists/*; \ @@ -165,7 +166,9 @@ RUN set -ex; \ fi; \ \ # some of the steps above generate a lot of "*.pyc" files (and setting "PYTHONDONTWRITEBYTECODE" beforehand doesn't propagate properly for some reason), so we clean them up manually (as long as they aren't owned by a package) - find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' + + find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +; \ + \ + postgres --version # make the sample config easier to munge (and "correct by default") RUN set -eux; \ @@ -177,14 +180,15 @@ RUN set -eux; \ RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql -ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin ENV PGDATA /var/lib/postgresql/data # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" VOLUME /var/lib/postgresql/data COPY docker-entrypoint.sh /usr/local/bin/ +{{ if .major >= 11 then "" else ( -}} RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat +{{ ) end -}} ENTRYPOINT ["docker-entrypoint.sh"] # We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL diff --git a/apply-templates.sh b/apply-templates.sh new file mode 100755 index 0000000000..58c8f441cb --- /dev/null +++ b/apply-templates.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +[ -f versions.json ] # run "versions.sh" first + +cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" + +jqt='.jq-template.awk' +if [ -n "${BASHBREW_SCRIPTS:-}" ]; then + jqt="$BASHBREW_SCRIPTS/jq-template.awk" +elif [ "$BASH_SOURCE" -nt "$jqt" ]; then + # https://github.com/docker-library/bashbrew/blob/master/scripts/jq-template.awk + wget -qO "$jqt" 'https://github.com/docker-library/bashbrew/raw/00e281f36edd19f52541a6ba2f215cc3c4645128/scripts/jq-template.awk' +fi + +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi + +generated_warning() { + cat <<-EOH + # + # NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" + # + # PLEASE DO NOT EDIT IT DIRECTLY. + # + + EOH +} + +for version; do + export version + + major="$(jq -r '.[env.version].major' versions.json)" + + variants="$(jq -r '.[env.version].debianSuites + ["alpine"] | map(@sh) | join(" ")' versions.json)" + eval "variants=( $variants )" + + for variant in "${variants[@]}"; do + export variant + + dir="$version/$variant" + mkdir -p "$dir" + + echo "processing $dir ..." + + if [ "$variant" = 'alpine' ]; then + template='Dockerfile-alpine.template' + else + template='Dockerfile-debian.template' + fi + { + generated_warning + gawk -f "$jqt" "$template" + } > "$dir/Dockerfile" + + cp -a docker-entrypoint.sh "$dir/" + if [ "$major" = '9' ]; then + sed -i -e 's/WALDIR/XLOGDIR/g' -e 's/waldir/xlogdir/g' "$dir/docker-entrypoint.sh" + fi + if [ "$variant" = 'alpine' ]; then + sed -i -e 's/gosu/su-exec/g' "$dir/docker-entrypoint.sh" + fi + done +done diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 00c9090aa8..4ecfc9527a 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -1,5 +1,5 @@ -#!/bin/bash -set -eu +#!/usr/bin/env bash +set -Eeuo pipefail declare -A aliases=( [13]='latest' @@ -9,11 +9,13 @@ declare -A aliases=( self="$(basename "$BASH_SOURCE")" cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -versions=( */ ) -versions=( "${versions[@]%/}" ) +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi # sort version numbers with highest first -IFS=$'\n'; versions=( $(echo "${versions[*]}" | sort -rV) ); unset IFS +IFS=$'\n'; set -- $(sort -rV <<<"$*"); unset IFS # get the most recent commit which modified any of "$@" fileCommit() { @@ -25,15 +27,19 @@ dirCommit() { local dir="$1"; shift ( cd "$dir" - fileCommit \ - Dockerfile \ - $(git show HEAD:./Dockerfile | awk ' + files="$( + git show HEAD:./Dockerfile | awk ' toupper($1) == "COPY" { for (i = 2; i < NF; i++) { + if ($i ~ /^--from=/) { + next + } print $i } } - ') + ' + )" + fileCommit Dockerfile $files ) } @@ -68,12 +74,16 @@ join() { echo "${out#$sep}" } -for version in "${versions[@]}"; do - commit="$(dirCommit "$version")" +for version; do + export version - pgdgVersion="$(git show "$commit":"$version/Dockerfile" | awk '$1 == "ENV" && $2 == "PG_VERSION" { print $3; exit }')" - fullVersion="${pgdgVersion%%-*}" - fullVersion="${fullVersion//'~'/-}" + variants="$(jq -r '.[env.version].debianSuites + ["alpine"] | map(@sh) | join(" ")' versions.json)" + eval "variants=( $variants )" + + debian="$(jq -r '.[env.version].debian' versions.json)" + + fullVersion="$(jq -r '.[env.version].version' versions.json)" + origVersion="$fullVersion" versionAliases=() while [ "$fullVersion" != "$version" -a "${fullVersion%[.-]*}" != "$fullVersion" ]; do @@ -83,42 +93,37 @@ for version in "${versions[@]}"; do # skip unadorned "version" on prereleases: https://www.postgresql.org/developer/beta/ # - https://github.com/docker-library/postgres/issues/662 # - https://github.com/docker-library/postgres/issues/784 - case "$pgdgVersion" in - *alpha* | *beta*| *rc*) ;; + case "$origVersion" in + *alpha* | *beta* | *rc*) ;; *) versionAliases+=( $version ) ;; esac versionAliases+=( ${aliases[$version]:-} ) - versionParent="$(awk 'toupper($1) == "FROM" { print $2 }' "$version/Dockerfile")" - versionArches="${parentRepoToArches[$versionParent]}" - - echo - cat <<-EOE - Tags: $(join ', ' "${versionAliases[@]}") - Architectures: $(join ', ' $versionArches) - GitCommit: $commit - Directory: $version - EOE - - for variant in alpine; do - [ -f "$version/$variant/Dockerfile" ] || continue + for variant in "${variants[@]}"; do + dir="$version/$variant" + commit="$(dirCommit "$dir")" - commit="$(dirCommit "$version/$variant")" + parent="$(awk 'toupper($1) == "FROM" { print $2 }' "$dir/Dockerfile")" + arches="${parentRepoToArches[$parent]}" variantAliases=( "${versionAliases[@]/%/-$variant}" ) variantAliases=( "${variantAliases[@]//latest-/}" ) - variantParent="$(awk 'toupper($1) == "FROM" { print $2 }' "$version/$variant/Dockerfile")" - variantArches="${parentRepoToArches[$variantParent]}" + if [ "$variant" = "$debian" ]; then + variantAliases=( + "${versionAliases[@]}" + "${variantAliases[@]}" + ) + fi echo cat <<-EOE Tags: $(join ', ' "${variantAliases[@]}") - Architectures: $(join ', ' $variantArches) + Architectures: $(join ', ' $arches) GitCommit: $commit - Directory: $version/$variant + Directory: $dir EOE done done diff --git a/update.sh b/update.sh index 45874c955c..bac2d7581c 100755 --- a/update.sh +++ b/update.sh @@ -1,165 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash set -Eeuo pipefail cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -versions=( "$@" ) -if [ ${#versions[@]} -eq 0 ]; then - versions=( */ ) -fi -versions=( "${versions[@]%/}" ) - -defaultDebianSuite='buster-slim' -declare -A debianSuite=( - # https://github.com/docker-library/postgres/issues/582 - [9.6]='stretch-slim' - [10]='stretch-slim' - [11]='stretch-slim' -) -defaultAlpineVersion='3.14' -declare -A alpineVersion=( - #[9.6]='3.5' -) - -packagesBase='http://apt.postgresql.org/pub/repos/apt/dists/' -declare -A suitePackageList=() suiteVersionPackageList=() suiteArches=() -_raw_package_list() { - local suite="$1"; shift - local component="$1"; shift - local arch="$1"; shift - - curl -fsSL "$packagesBase/$suite-pgdg/$component/binary-$arch/Packages.bz2" | bunzip2 -} -fetch_suite_package_list() { - local suite="$1"; shift - local version="$1"; shift - local arch="$1"; shift - - # normal (GA) releases end up in the "main" component of upstream's repository - if [ -z "${suitePackageList["$suite-$arch"]:+isset}" ]; then - local suiteArchPackageList - suiteArchPackageList="$(_raw_package_list "$suite" 'main' "$arch")" - suitePackageList["$suite-$arch"]="$suiteArchPackageList" - fi - - # ... but pre-release versions (betas, etc) end up in the "PG_MAJOR" component (so we need to check both) - if [ -z "${suiteVersionPackageList["$suite-$version-$arch"]:+isset}" ]; then - local versionPackageList - versionPackageList="$(_raw_package_list "$suite" "$version" "$arch")" - suiteVersionPackageList["$suite-$version-$arch"]="$versionPackageList" - fi -} -awk_package_list() { - local suite="$1"; shift - local version="$1"; shift - local arch="$1"; shift - - awk -F ': ' -v version="$version" "$@" <<<"${suitePackageList["$suite-$arch"]}"$'\n'"${suiteVersionPackageList["$suite-$version-$arch"]}" -} -fetch_suite_arches() { - local suite="$1"; shift - - if [ -z "${suiteArches["$suite"]:+isset}" ]; then - local suiteRelease - suiteRelease="$(curl -fsSL "$packagesBase/$suite-pgdg/Release")" - suiteArches["$suite"]="$(gawk <<<"$suiteRelease" -F ':[[:space:]]+' '$1 == "Architectures" { print $2; exit }')" - fi -} - -for version in "${versions[@]}"; do - tag="${debianSuite[$version]:-$defaultDebianSuite}" - suite="${tag%%-slim}" - majorVersion="${version%%.*}" - - fetch_suite_package_list "$suite" "$version" 'amd64' - fullVersion="$( - awk_package_list "$suite" "$version" 'amd64' ' - $1 == "Package" { pkg = $2 } - $1 == "Version" && pkg == "postgresql-" version { print $2; exit } - ' - )" - if [ -z "$fullVersion" ]; then - echo >&2 "error: missing postgresql-$version package!" - exit 1 - fi - - fetch_suite_arches "$suite" - versionArches= - for arch in ${suiteArches["$suite"]}; do - fetch_suite_package_list "$suite" "$version" "$arch" - archVersion="$( - awk_package_list "$suite" "$version" "$arch" ' - $1 == "Package" { pkg = $2 } - $1 == "Version" && pkg == "postgresql-" version { print $2; exit } - ' - )" - if [ "$archVersion" = "$fullVersion" ]; then - [ -z "$versionArches" ] || versionArches+=' | ' - versionArches+="$arch" - fi - done - - echo "$version: $fullVersion ($versionArches)" - - cp docker-entrypoint.sh "$version/" - sed -e 's/%%PG_MAJOR%%/'"$version"'/g;' \ - -e 's/%%PG_VERSION%%/'"$fullVersion"'/g' \ - -e 's/%%DEBIAN_TAG%%/'"$tag"'/g' \ - -e 's/%%DEBIAN_SUITE%%/'"$suite"'/g' \ - -e 's/%%ARCH_LIST%%/'"$versionArches"'/g' \ - Dockerfile-debian.template \ - > "$version/Dockerfile" - if [ "$majorVersion" = '9' ]; then - sed -i -e 's/WALDIR/XLOGDIR/g' \ - -e 's/waldir/xlogdir/g' \ - "$version/docker-entrypoint.sh" - # ICU support was introduced in PostgreSQL 10 (https://www.postgresql.org/docs/10/static/release-10.html#id-1.11.6.9.5.13) - sed -i -e '/icu/d' "$version/Dockerfile" - else - # postgresql-contrib-10 package does not exist, but is provided by postgresql-10 - # Packages.gz: - # Package: postgresql-10 - # Provides: postgresql-contrib-10 - sed -i -e '/postgresql-contrib-/d' "$version/Dockerfile" - fi - - if [ "$majorVersion" != '13' ]; then - sed -i -e '/DEBIAN_FRONTEND/d' "$version/Dockerfile" - fi - - # TODO figure out what to do with odd version numbers here, like release candidates - srcVersion="${fullVersion%%-*}" - # change "10~beta1" to "10beta1" for ftp urls - tilde='~' - srcVersion="${srcVersion//$tilde/}" - srcSha256="$(curl -fsSL "https://ftp.postgresql.org/pub/source/v${srcVersion}/postgresql-${srcVersion}.tar.bz2.sha256" | cut -d' ' -f1)" - for variant in alpine; do - if [ ! -d "$version/$variant" ]; then - continue - fi - - cp docker-entrypoint.sh "$version/$variant/" - sed -i 's/gosu/su-exec/g' "$version/$variant/docker-entrypoint.sh" - sed -e 's/%%PG_MAJOR%%/'"$version"'/g' \ - -e 's/%%PG_VERSION%%/'"$srcVersion"'/g' \ - -e 's/%%PG_SHA256%%/'"$srcSha256"'/g' \ - -e 's/%%ALPINE-VERSION%%/'"${alpineVersion[$version]:-$defaultAlpineVersion}"'/g' \ - "Dockerfile-$variant.template" \ - > "$version/$variant/Dockerfile" - if [ "$majorVersion" = '9' ]; then - sed -i -e 's/WALDIR/XLOGDIR/g' \ - -e 's/waldir/xlogdir/g' \ - "$version/$variant/docker-entrypoint.sh" - # ICU support was introduced in PostgreSQL 10 (https://www.postgresql.org/docs/10/static/release-10.html#id-1.11.6.9.5.13) - sed -i -e '/icu/d' "$version/$variant/Dockerfile" - fi - - if [ "$majorVersion" -gt 11 ]; then - sed -i '/backwards compat/d' "$version/$variant/Dockerfile" - fi - if [ "$majorVersion" -lt 11 ]; then - # JIT / LLVM is only supported in PostgreSQL 11+ (https://github.com/docker-library/postgres/issues/475) - sed -i '/llvm/d' "$version/$variant/Dockerfile" - fi - done -done +./versions.sh "$@" +./apply-templates.sh "$@" diff --git a/versions.json b/versions.json new file mode 100644 index 0000000000..d389bc37f2 --- /dev/null +++ b/versions.json @@ -0,0 +1,124 @@ +{ + "10": { + "alpine": "3.14", + "buster": { + "arches": [ + "amd64", + "arm64", + "i386", + "ppc64el" + ], + "version": "10.17-1.pgdg100+1" + }, + "debian": "stretch", + "debianSuites": [ + "buster", + "stretch" + ], + "major": 10, + "sha256": "5af28071606c9cd82212c19ba584657a9d240e1c4c2da28fc1f3998a2754b26c", + "stretch": { + "arches": [ + "amd64", + "i386", + "ppc64el" + ], + "version": "10.17-1.pgdg90+1" + }, + "version": "10.17" + }, + "11": { + "alpine": "3.14", + "buster": { + "arches": [ + "amd64", + "arm64", + "i386", + "ppc64el" + ], + "version": "11.12-1.pgdg100+1" + }, + "debian": "stretch", + "debianSuites": [ + "buster", + "stretch" + ], + "major": 11, + "sha256": "87f9d8b16b2b8ef71586f2ec76beac844819f64734b07fa33986755c2f53cb04", + "stretch": { + "arches": [ + "amd64", + "i386", + "ppc64el" + ], + "version": "11.12-1.pgdg90+1" + }, + "version": "11.12" + }, + "12": { + "alpine": "3.14", + "buster": { + "arches": [ + "amd64", + "arm64", + "i386", + "ppc64el" + ], + "version": "12.7-1.pgdg100+1" + }, + "debian": "buster", + "debianSuites": [ + "buster" + ], + "major": 12, + "sha256": "8490741f47c88edc8b6624af009ce19fda4dc9b31c4469ce2551d84075d5d995", + "version": "12.7" + }, + "13": { + "alpine": "3.14", + "buster": { + "arches": [ + "amd64", + "arm64", + "i386", + "ppc64el" + ], + "version": "13.3-1.pgdg100+1" + }, + "debian": "buster", + "debianSuites": [ + "buster" + ], + "major": 13, + "sha256": "3cd9454fa8c7a6255b6743b767700925ead1b9ab0d7a0f9dcb1151010f8eb4a1", + "version": "13.3" + }, + "9.6": { + "alpine": "3.14", + "buster": { + "arches": [ + "amd64", + "arm64", + "i386", + "ppc64el" + ], + "version": "9.6.22-1.pgdg100+1" + }, + "debian": "stretch", + "debianSuites": [ + "buster", + "stretch" + ], + "major": 9, + "sha256": "3d32cd101025a0556813397c69feff3df3d63736adb8adeaf365c522f39f2930", + "stretch": { + "arches": [ + "amd64", + "i386", + "ppc64el" + ], + "version": "9.6.22-1.pgdg90+1" + }, + "version": "9.6.22" + } +} diff --git a/versions.sh b/versions.sh new file mode 100755 index 0000000000..3d2cd02d9b --- /dev/null +++ b/versions.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# https://github.com/docker-library/postgres/issues/582 😬 +defaultDebianSuite='buster' +declare -A debianSuites=( + [9.6]='stretch' + [10]='stretch' + [11]='stretch' +) +allDebianSuites=( + buster + stretch +) +defaultAlpineVersion='3.14' +declare -A alpineVersions=( + #[9.6]='3.5' +) + +cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" + +versions=( "$@" ) +if [ ${#versions[@]} -eq 0 ]; then + versions=( */ ) + json='{}' +else + json="$(< versions.json)" +fi +versions=( "${versions[@]%/}" ) + +packagesBase='http://apt.postgresql.org/pub/repos/apt/dists/' +declare -A suitePackageList=() suiteVersionPackageList=() suiteArches=() +_raw_package_list() { + local suite="$1"; shift + local component="$1"; shift + local arch="$1"; shift + + curl -fsSL "$packagesBase/$suite-pgdg/$component/binary-$arch/Packages.bz2" | bunzip2 +} +fetch_suite_package_list() { + local suite="$1"; shift + local version="$1"; shift + local arch="$1"; shift + + # normal (GA) releases end up in the "main" component of upstream's repository + if [ -z "${suitePackageList["$suite-$arch"]:+isset}" ]; then + local suiteArchPackageList + suiteArchPackageList="$(_raw_package_list "$suite" 'main' "$arch")" + suitePackageList["$suite-$arch"]="$suiteArchPackageList" + fi + + # ... but pre-release versions (betas, etc) end up in the "PG_MAJOR" component (so we need to check both) + if [ -z "${suiteVersionPackageList["$suite-$version-$arch"]:+isset}" ]; then + local versionPackageList + versionPackageList="$(_raw_package_list "$suite" "$version" "$arch")" + suiteVersionPackageList["$suite-$version-$arch"]="$versionPackageList" + fi +} +awk_package_list() { + local suite="$1"; shift + local version="$1"; shift + local arch="$1"; shift + + awk -F ': ' -v version="$version" "$@" <<<"${suitePackageList["$suite-$arch"]}"$'\n'"${suiteVersionPackageList["$suite-$version-$arch"]}" +} +fetch_suite_arches() { + local suite="$1"; shift + + if [ -z "${suiteArches["$suite"]:+isset}" ]; then + local suiteRelease + suiteRelease="$(curl -fsSL "$packagesBase/$suite-pgdg/Release")" + suiteArches["$suite"]="$(gawk <<<"$suiteRelease" -F ':[[:space:]]+' '$1 == "Architectures" { print $2; exit }')" + fi +} + +for version in "${versions[@]}"; do + export version + + versionAlpineVersion="${alpineVersions[$version]:-$defaultAlpineVersion}" + versionDebianSuite="${debianSuites[$version]-$defaultDebianSuite}" # intentionally missing ":" so it can be empty (again, https://github.com/docker-library/postgres/issues/582 😭) + export versionAlpineVersion versionDebianSuite + + doc="$(jq -nc '{ + alpine: env.versionAlpineVersion, + debian: env.versionDebianSuite, + }')" + + versionDebianSuites=() + for suite in "${allDebianSuites[@]}"; do + versionDebianSuites+=( "$suite" ) + if [ "$suite" = "$versionDebianSuite" ]; then + # if our default is "buster" we shouldn't even consider "stretch" + break + fi + done + + fullVersion= + for suite in "${versionDebianSuites[@]}"; do + fetch_suite_package_list "$suite" "$version" 'amd64' + suiteVersion="$(awk_package_list "$suite" "$version" 'amd64' ' + $1 == "Package" { pkg = $2 } + $1 == "Version" && pkg == "postgresql-" version { print $2; exit } + ')" + srcVersion="${suiteVersion%%-*}" + tilde='~' + srcVersion="${srcVersion//$tilde/}" + [ -n "$fullVersion" ] || fullVersion="$srcVersion" + if [ "$fullVersion" != "$srcVersion" ]; then + echo >&2 "warning: $version should be '$fullVersion' but $suite is '$srcVersion'" + continue + fi + + versionArches='[]' + fetch_suite_arches "$suite" + for arch in ${suiteArches["$suite"]}; do + fetch_suite_package_list "$suite" "$version" "$arch" + archVersion="$(awk_package_list "$suite" "$version" "$arch" ' + $1 == "Package" { pkg = $2 } + $1 == "Version" && pkg == "postgresql-" version { print $2; exit } + ')" + if [ "$archVersion" = "$suiteVersion" ]; then + versionArches="$(jq <<<"$versionArches" -c --arg arch "$arch" '. += [$arch]')" + fi + done + + export suite suiteVersion + doc="$(jq <<<"$doc" -c --argjson arches "$versionArches" ' + .[env.suite] = { + version: env.suiteVersion, + arches: $arches, + } + | .debianSuites += [ env.suite ] + ')" + done + + sha256="$( + curl -fsSL "https://ftp.postgresql.org/pub/source/v${fullVersion}/postgresql-${fullVersion}.tar.bz2.sha256" \ + | cut -d' ' -f1 + )" + + echo "$version: $fullVersion" + + export fullVersion sha256 major="${version%%.*}" + json="$(jq <<<"$json" -c --argjson doc "$doc" ' + .[env.version] = ($doc + { + version: env.fullVersion, + sha256: env.sha256, + major: (env.major | tonumber), + }) + ')" +done + +jq <<<"$json" -S . > versions.json