ci(macos): Avoid linking against homebrew

Homebrew binaries are always compiled for exactly the version they're
installed on making them very un-portable. When a wheel is "repaired" by
cibuildwheel, delocate-wheel pulls in _psycopg's dependencies
(libpq.dylib, libssl.dylib and libcrypto.dylib) which, on a GitHub
Actions macOS 14 runner, are provided by Homebrew and are therefore only
macOS >= 14 compatible. The resultant wheel is therefore incompatible
with all but the latest macOS versions.

Build all dependencies from source so that we can set the deployment
target to something sensible. Fixes #1753.

Author: bwoodsend

.github/workflows/packages.yml

@@ -5,10 +5,12 @@ on:
 
 env:
   PIP_BREAK_SYSTEM_PACKAGES: "1"
+  LIBPQ_VERSION: "16.0"
+  OPENSSL_VERSION: "1.1.1w"
 
 jobs:
   build-sdist:
-    if: true
+    if: false
     strategy:
       fail-fast: false
       matrix:
@@ -58,16 +60,12 @@ jobs:
   build-linux:
     if: true
 
-    env:
-      LIBPQ_VERSION: "16.0"
-      OPENSSL_VERSION: "1.1.1w"
-
     strategy:
       fail-fast: false
       matrix:
         platform: [manylinux, musllinux]
-        arch: [x86_64, i686, aarch64, ppc64le]
-        pyver: [cp38, cp39, cp310, cp311, cp312, cp313]
+        arch: [x86_64] #, i686, aarch64, ppc64le]
+        pyver: [cp38] #, cp39, cp310, cp311, cp312, cp313]
 
     runs-on: ubuntu-latest
     steps:
@@ -140,18 +138,12 @@ jobs:
       matrix:
         # These archs require an Apple M1 runner: [arm64, universal2]
         arch: [x86_64]
-        pyver: [cp38, cp39, cp310, cp311, cp312, cp313]
+        pyver: [cp38, cp313]
         macver: ["12"]
         include:
           - arch: arm64
             pyver: cp310
             macver: "14"
-          - arch: arm64
-            pyver: cp311
-            macver: "14"
-          - arch: arm64
-            pyver: cp312
-            macver: "14"
           - arch: arm64
             pyver: cp313
             macver: "14"
@@ -160,6 +152,12 @@ jobs:
       - name: Checkout repos
         uses: actions/checkout@v4
 
+      - name: Build dependencies
+        run: ./scripts/build/build_libpq.sh
+
+      - name: Show dependency tree
+        run: otool -L /tmp/libpq.build/lib/*.dylib
+
       - name: Build wheels
         uses: pypa/[email protected]
         env:
@@ -170,12 +168,11 @@ jobs:
             export PYTHONPATH={project} &&
             python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
           CIBW_ENVIRONMENT: >-
-            MACOSX_DEPLOYMENT_TARGET=${{ matrix.macver }}.0
             PG_VERSION=16
             PACKAGE_NAME=psycopg2-binary
             PSYCOPG2_TESTDB=postgres
             PSYCOPG2_TEST_FAST=1
-            PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH"
+            PATH="/tmp/libpq.build/bin:$PATH"
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4

.github/workflows/tests.yml

@@ -9,6 +9,7 @@ on:
 
 jobs:
   tests:
+    if: false
     name: Unit tests run
     runs-on: ubuntu-latest
 

scripts/build/build_libpq.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Build a modern version of libpq and depending libs from source on Centos 5
+# Build a modern version of libpq and depending libs from source on Centos 5, Alpine or macOS
 
 set -euo pipefail
 set -x
@@ -12,30 +12,69 @@ postgres_version="${LIBPQ_VERSION}"
 # last release: https://www.openssl.org/source/
 openssl_version="${OPENSSL_VERSION}"
 
+# last release: https://kerberos.org/dist/
+krb5_version="1.21.3"
+
+# last release: https://www.gnu.org/software/gettext/
+gettext_version="0.22.5"
+
 # last release: https://openldap.org/software/download/
-ldap_version="2.6.3"
+ldap_version="2.6.8"
 
 # last release: https://github.com/cyrusimap/cyrus-sasl/releases
 sasl_version="2.1.28"
 
 export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build}
 
-if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then
+case "$(uname)" in
+    Darwin)
+        ID=macos
+        library_suffix=dylib
+        ;;
+
+    Linux)
+        source /etc/os-release
+        library_suffix=so
+        ;;
+
+    *)
+        echo "$0: unexpected Operating system: '$(uname)'" >&2
+        exit 1
+        ;;
+esac
+
+if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.${library_suffix}" ]]; then
     echo "libpq already available: build skipped" >&2
     exit 0
 fi
 
-source /etc/os-release
-
 case "$ID" in
     centos)
         yum update -y
         yum install -y zlib-devel krb5-devel pam-devel
+        curl="$(which curl)"
         ;;
 
     alpine)
         apk upgrade
         apk add --no-cache zlib-dev krb5-dev linux-pam-dev openldap-dev openssl-dev
+        curl="$(which curl)"
+        ;;
+
+    macos)
+        brew install automake
+        # If available, libpq seemingly insists on linking against homebrew's
+        # openssl no matter what so remove it. Since homebrew's curl depends on
+        # it, force use of system curl.
+        brew uninstall --force --ignore-dependencies openssl m4
+        curl="/usr/bin/curl"
+        # The deployment target should be <= to that of the oldest supported Python version.
+        # e.g. https://www.python.org/downloads/release/python-380/
+        if [ "$(uname -m)" == "x86_64" ]; then
+            export MACOSX_DEPLOYMENT_TARGET=10.9
+        else
+            export MACOSX_DEPLOYMENT_TARGET=11.0
+        fi
         ;;
 
     *)
@@ -44,12 +83,12 @@ case "$ID" in
         ;;
 esac
 
-if [ "$ID" == "centos" ]; then
+if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
 
     # Build openssl if needed
     openssl_tag="OpenSSL_${openssl_version//./_}"
     openssl_dir="openssl-${openssl_tag}"
-    if [ ! -d "${openssl_dir}" ]; then curl -sL \
+    if [ ! -d "${openssl_dir}" ]; then "$curl" -sL \
             https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \
             | tar xzf -
 
@@ -70,7 +109,55 @@ if [ "$ID" == "centos" ]; then
 fi
 
 
-if [ "$ID" == "centos" ]; then
+if [ "$ID" == "macos" ]; then
+
+    # Build kerberos if needed
+    krb5_dir="krb5-${krb5_version}/src"
+    if [ ! -d "${krb5_dir}" ]; then
+        "$curl" -sL \
+            curl -sL "https://kerberos.org/dist/krb5/$(echo 1.21.3 | grep -oE '\d+\.\d+')/krb5-${krb5_version}.tar.gz" \
+            | tar xzf -
+
+        cd "${krb5_dir}"
+
+        ./configure --prefix=${LIBPQ_BUILD_PREFIX} \
+            CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+        make
+    else
+        cd "${krb5_dir}"
+    fi
+
+    make install
+    cd ../..
+
+fi
+
+
+if [ "$ID" == "macos" ]; then
+
+    # Build gettext if needed
+    gettext_dir="gettext-${gettext_version}"
+    if [ ! -d "${gettext_dir}" ]; then
+        "$curl" -sL \
+            curl -sL "https://ftp.gnu.org/pub/gnu/gettext/gettext-${gettext_version}.tar.gz" \
+            | tar xzf -
+
+        cd "${gettext_dir}"
+
+        ./configure --prefix=${LIBPQ_BUILD_PREFIX} \
+            CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+        make -C gettext-runtime all
+    else
+        cd "${gettext_dir}"
+    fi
+
+    make -C gettext-runtime install
+    cd ..
+
+fi
+
+
+if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
 
     # Build libsasl2 if needed
     # The system package (cyrus-sasl-devel) causes an amazing error on i686:
@@ -79,14 +166,14 @@ if [ "$ID" == "centos" ]; then
     sasl_tag="cyrus-sasl-${sasl_version}"
     sasl_dir="cyrus-sasl-${sasl_tag}"
     if [ ! -d "${sasl_dir}" ]; then
-        curl -sL \
+        "$curl" -sL \
             https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \
             | tar xzf -
 
         cd "${sasl_dir}"
 
         autoreconf -i
-        ./configure --prefix=${LIBPQ_BUILD_PREFIX} \
+        ./configure --prefix=${LIBPQ_BUILD_PREFIX} --disable-macos-framework \
             CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
         make
     else
@@ -102,13 +189,13 @@ if [ "$ID" == "centos" ]; then
 fi
 
 
-if [ "$ID" == "centos" ]; then
+if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
 
     # Build openldap if needed
     ldap_tag="${ldap_version}"
     ldap_dir="openldap-${ldap_tag}"
     if [ ! -d "${ldap_dir}" ]; then
-        curl -sL \
+        "$curl" -sL \
             https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \
             | tar xzf -
 
@@ -129,7 +216,7 @@ if [ "$ID" == "centos" ]; then
     make -C libraries/liblber/ install
     make -C libraries/libldap/ install
     make -C include/ install
-    chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.so*
+    chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.${library_suffix}*
     cd ..
 
 fi
@@ -139,17 +226,19 @@ fi
 postgres_tag="REL_${postgres_version//./_}"
 postgres_dir="postgres-${postgres_tag}"
 if [ ! -d "${postgres_dir}" ]; then
-    curl -sL \
+    "$curl" -sL \
         https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \
         | tar xzf -
 
     cd "${postgres_dir}"
 
-    # Match the default unix socket dir default with what defined on Ubuntu and
-    # Red Hat, which seems the most common location
-    sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\
+    if [ "$ID" != "macos" ]; then
+        # Match the default unix socket dir default with what defined on Ubuntu and
+        # Red Hat, which seems the most common location
+        sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\
 '|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \
-        src/include/pg_config_manual.h
+            src/include/pg_config_manual.h
+    fi
 
     # Often needed, but currently set by the workflow
     # export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib"
@@ -171,4 +260,4 @@ make -C src/bin/pg_config install
 make -C src/include install
 cd ..
 
-find ${LIBPQ_BUILD_PREFIX} -name \*.so.\* -type f -exec strip --strip-unneeded {} \;
+find ${LIBPQ_BUILD_PREFIX} -name \*.${library_suffix}.\* -type f -exec strip --strip-unneeded {} \;