More cleanup

.github/workflows/build_wasm_postgres.yml

@@ -0,0 +1,39 @@
+name: Build postgres-pglite (aka Postgresql WASM)
+
+permissions:
+  pull-requests: write
+  issues: write
+  contents: read
+
+on:
+  workflow_call:
+  push:
+    branches: ['pglite/REL_16_STABLE']
+  pull_request:
+
+jobs:
+  build:
+    name: Build postgres-pglite (WASM Postgres)
+    runs-on: ubuntu-22.04
+    env:
+      BUILD_CONFIG_FILE: pglite/.buildconfig
+      PGROOT: /tmp/pglite
+      TOTAL_MEMORY: 128MB
+      CMA_MB: 16
+      DEBUG: false
+      OBJDUMP: true
+      contrib: contrib
+      extra: extra
+      EXTRA_EXT: vector
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Build postgres WASM via emsdk
+        working-directory: ${{ github.workspace }}
+        env: 
+          PGSRC: ${{ github.workspace }}
+        run: |
+          pglite/cibuild/build-with-docker.sh

configure

@@ -19446,7 +19446,7 @@ ac_config_files="$ac_config_files GNUmakefile src/Makefile.global"
 
 
 ac_config_links="$ac_config_links src/backend/port/pg_sema.c:${SEMA_IMPLEMENTATION} src/backend/port/pg_shmem.c:${SHMEM_IMPLEMENTATION} src/include/pg_config_os.h:src/include/port/${template}.h src/Makefile.port:src/makefiles/Makefile.${template}"
-
+ac_config_links="$ac_config_links src/include/port/wasm_common.h:src/include/port/wasm_common.h"
 
 if test "$PORTNAME" = "win32"; then
 ac_config_commands="$ac_config_commands check_win32_symlinks"

pglite/cibuild.sh

@@ -240,15 +240,6 @@ END
 
     . ${PGROOT}/pgopts.sh
 
-    # make sure no non-mvp feature gets in.
-    cat > ${PGROOT}/config.site <<END
-pgac_cv_sse42_crc32_intrinsics_=no
-pgac_cv_sse42_crc32_intrinsics__msse4_2=no
-pgac_sse42_crc32_intrinsics=no
-pgac_armv8_crc32c_intrinsics=no
-ac_cv_search_sem_open=no
-END
-
 
     # workaround no "locale -a" for Node.
     # this is simply the minimal result a popen call would give.

pglite/cibuild/build-with-docker.sh

@@ -15,14 +15,9 @@ fi
 IMG_NAME="electricsql/pglite-builder"
 IMG_TAG="${PG_VERSION}_${SDK_VERSION}"
 
-SDK_ARCHIVE="${SDK_ARCHIVE:-python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4}"
-WASI_SDK_ARCHIVE="${WASI_SDK_ARCHIVE:-python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4}"
-
 docker run \
   --rm \
   -e OBJDUMP=${OBJDUMP:-true} \
-  -e SDK_ARCHIVE \
-  -e WASI_SDK_ARCHIVE \
   -e PGSRC=/workspace/postgres-src \
   -e POSTGRES_PGLITE_OUT=/workspace/dist \
   -v ./pglite/cibuild.sh:/workspace/cibuild.sh:rw \

pglite/cibuild/pg-16.x.sh

@@ -65,6 +65,11 @@ then
 "
 else
     echo "Building $ARCHIVE (patched) from $PGSRC"
-    . cibuild/pgbuild.sh
+    if ${WASI}
+    then
+        . cibuild/pgbuild.sh
+    else
+        . cibuild/pgbuild.wasm.sh
+    fi
 fi
 

pglite/cibuild/pg-git.sh

@@ -69,7 +69,12 @@ else
 
     else
         echo "Building $ARCHIVE (patched) from $PGSRC"
-        . cibuild/pgbuild.sh
+        if ${WASI}
+        then
+            . cibuild/pgbuild.sh
+        else
+            . cibuild/pgbuild.wasm.sh
+        fi
     fi
 fi
 

pglite/cibuild/pgbuild.sh

@@ -93,35 +93,32 @@ CC_PGLITE=$CC_PGLITE
     if $WASI
     then
         export EXT=wasi
-        cat > ${PGROOT}/config.site <<END
-ac_cv_exeext=.wasi
-END
+        ACCVEXEEXT==.wasi
+
         if $GETZIC
         then
             cat > bin/zic <<END
 #!/bin/bash
 #. /opt/python-wasm-sdk/wasm32-wasi-shell.sh
-TZ=UTC PGTZ=UTC $(command -v wasi-run) $(pwd)/src/timezone/zic.wasi \$@
+TZ=UTC PGTZ=UTC $(which wasi-run) $(pwd)/src/timezone/zic.wasi \$@
 END
         fi
 
     else
         export EXT=wasm
-        cat > ${PGROOT}/config.site <<END
-ac_cv_exeext=.cjs
-END
+        ACCVEXEEXT=.cjs
 
         if $GETZIC
         then
             cat > bin/zic <<END
 #!/bin/bash
 #. /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh
-TZ=UTC PGTZ=UTC $(command -v node) $(pwd)/src/timezone/zic.cjs \$@
+TZ=UTC PGTZ=UTC $(which node) $(pwd)/src/timezone/zic.cjs \$@
 END
         fi
     fi
 
-    if EM_PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig CONFIG_SITE=${PGROOT}/config.site emconfigure $CNF --with-template=$BUILD
+    if EM_PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig ac_cv_exeext=$ACCVEXEEXT emconfigure $CNF --with-template=$BUILD
     then
         echo configure ok
     else
@@ -254,7 +251,7 @@ END
 	mv -vf ./src/bin/pg_resetwal/pg_resetwal.$EXT  ./src/bin/initdb/initdb.$EXT ./src/backend/postgres.$EXT ${PGROOT}/bin/
 
 
-    python3 > ${PGROOT}/PGPASSFILE <<END
+    cat > ${PGROOT}/PGPASSFILE <<END
 USER="${PGPASS:-postgres}"
 PASS="${PGUSER:-postgres}"
 md5pass =  "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest()
@@ -308,50 +305,6 @@ END
     # for extensions building
     chmod +x ${PGROOT}/bin/pg_config
 
-
-	echo "initdb for PGDATA=${PGDATA} "
-
-    # create empty db hack
-
-	cat >$PGROOT/initdb.sh <<END
-#!/bin/bash
-rm -rf ${PGDATA} /tmp/initdb-* ${PGROOT}/wal/*
-export TZ=UTC
-export PGTZ=UTC
-SQL=/tmp/initdb-\$\$
-# TODO: --waldir=${PGROOT}/wal
-> /tmp/initdb.txt
-
-${PGROOT}/initdb --no-clean --wal-segsize=1 -g $LANG $CRED --pgdata=${PGDATA}
-
-mv /tmp/initdb.boot.txt \${SQL}.boot.sql
-mv /tmp/initdb.single.txt \${SQL}.single.sql
-
-if \${CI:-false}
-then
-    cp -vf \$SQL ${PGROOT}/\$(md5sum \$SQL|cut -c1-32).sql
-fi
-
-# --wal-segsize=1  -> -X 1048576
-
-# CKSUM_B -k --data-checksums
-# 2024-04-24 05:53:28.121 GMT [42] WARNING:  page verification failed, calculated checksum 5487 but expected 0
-# 2024-04-24 05:53:28.121 GMT [42] FATAL:  invalid page in block 0 of relation base/1/1259
-
-CMD="${PGROOT}/postgres --boot -D ${PGDATA} -d 3 $PGOPTS -X 1048576"
-echo "\$CMD < \$SQL.boot.sql"
-\$CMD < \$SQL.boot.sql 2>&1 \\
- | grep -v --line-buffered 'bootstrap> boot' \\
- | grep -v --line-buffered 'index'
-
-echo "
-
-\$(md5sum /tmp/initdb-\$\$.*.sql)
-
-    boot done
-"
-END
-
     popd
 echo "pgbuild:end
 

pglite/cibuild/pgbuild.wasm.sh

@@ -0,0 +1,120 @@
+echo "
+
+
+
+pgbuild.wasm:begin
+
+CC_PGLITE=$CC_PGLITE
+
+"
+    mkdir -p build/postgres
+    pushd build/postgres
+
+    # create empty package.json to avoid emsdk node conflicts
+    # with root package.json of project
+    echo "{}" > package.json
+
+    export MAIN_MODULE="-sMAIN_MODULE=1"
+    export XML2_CONFIG=$PREFIX/bin/xml2-config
+    export ZIC=$(pwd)/bin/zic
+    export EXT=wasm
+
+    cp -r ${PGSRC}/pglite/other/* /tmp/pglite/
+
+
+    CNF="${PGSRC}/configure FLEX=`which flex` --prefix=/tmp/pglite \
+ --cache-file=/tmp/pglite/config.cache.emscripten \
+ --disable-spinlocks --disable-largefile --without-llvm \
+ --without-pam --disable-largefile --with-openssl=no \
+ --without-readline --without-icu \
+ --with-uuid=ossp \
+ --with-zlib --with-libxml --with-libxslt \
+  ${PGDEBUG}"
+
+    echo "  ==== building wasm MVP:$MVP Debug=${PGDEBUG} with opts : $@  == "
+
+    mkdir -p bin
+
+    cat > bin/zic <<END
+#!/bin/bash
+TZ=UTC PGTZ=UTC node $(pwd)/src/timezone/zic.cjs \$@
+END
+
+    if ac_cv_exeext=.cjs emconfigure $CNF --with-template=emscripten
+    then
+        echo configure ok
+    else
+        echo configure failed
+        exit 76
+    fi
+
+    mkdir -p /tmp/pglite/bin
+
+    ln -sf /tmp/pglite/bin/emsdk-shared bin/emsdk-shared
+
+    chmod +x bin/zic /tmp/pglite/bin/emsdk-shared
+
+    # for zic and emsdk-shared/wasi-shared called from makefile
+    export PATH=$(pwd)/bin:$PATH
+
+    EMCC_NODE="-sEXIT_RUNTIME=1 -DEXIT_RUNTIME -sNODERAWFS -sENVIRONMENT=node"
+
+    # EMCC_ENV="${EMCC_NODE} -sFORCE_FILESYSTEM=0"
+    EMCC_ENV="${EMCC_NODE} -sERROR_ON_UNDEFINED_SYMBOLS"
+
+    # PREFIX only required for static initdb
+    EMCC_CFLAGS="-sERROR_ON_UNDEFINED_SYMBOLS=1 ${CC_PGLITE} -DPREFIX=/tmp/pglite -Wno-macro-redefined -Wno-unused-function"
+
+    # ZIC=${ZIC:-$(realpath bin/zic)}
+
+	if EMCC_CFLAGS="${EMCC_ENV} ${EMCC_CFLAGS}" emmake make emscripten=1 -j $(nproc) 2>&1 > /tmp/build.log
+	then
+        echo build ok
+        cp -vf src/backend/postgres src/backend/postgres.cjs
+
+        # if running a 32bits zic from current build
+        unset LD_PRELOAD
+
+        if EMCC_CFLAGS="${EMCC_ENV} ${EMCC_CFLAGS}" emmake make emscripten=1 install 2>&1 > /tmp/install.log
+        then
+            echo install ok
+            pushd /tmp/pglite
+
+            find . -type f | grep -v plpgsql > /tmp/pglite/pg.installed
+            popd
+
+            goback=$(pwd)
+            popd
+            python3 cibuild/pack_extension.py builtin
+            pushd $goback
+
+            pushd /tmp/pglite
+            find . -type f  > /tmp/pglite/pg.installed
+            popd
+
+        else
+            cat /tmp/install.log
+            echo "install failed"
+            exit 225
+        fi
+    else
+        cat /tmp/build.log
+        echo "build failed"
+        exit 230
+	fi
+
+    # wip
+    mv -vf ./src/bin/psql/psql.$EXT ./src/bin/pg_config/pg_config.$EXT /tmp/pglite/bin/
+    mv -vf ./src/bin/pg_dump/pg_restore.$EXT ./src/bin/pg_dump/pg_dump.$EXT ./src/bin/pg_dump/pg_dumpall.$EXT /tmp/pglite/bin/
+	mv -vf ./src/bin/pg_resetwal/pg_resetwal.$EXT  ./src/bin/initdb/initdb.$EXT ./src/backend/postgres.$EXT /tmp/pglite/bin/
+
+    # force node wasm version
+    cp -vf /tmp/pglite/postgres /tmp/pglite/bin/postgres && chmod +x /tmp/pglite/postgres /tmp/pglite/bin/postgres
+
+    popd
+echo "pgbuild:end
+
+
+
+
+"

pglite/cibuild/repo-pg-16.x.sh

@@ -15,4 +15,9 @@ rm postgresql 2>/dev/null
 ln -s $PGSRC postgresql
 
 echo "Building postgresql-${PG_VERSION} in folder $PGSRC"
-. cibuild/pgbuild.sh
+if ${WASI}
+then
+    . cibuild/pgbuild.sh
+else
+    . cibuild/pgbuild.wasm.sh
+fi

pglite/other/PGPASSFILE

@@ -0,0 +1,9 @@
+USER="postgres"
+PASS="postgres"
+md5pass =  "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest()
+print(f"localhost:5432:postgres:{USER}:{md5pass}")
+
+USER="login"
+PASS="password"
+md5pass =  "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest()
+print(f"localhost:5432:postgres:{USER}:{md5pass}")

pglite/other/bin/emsdk-shared

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "[$(pwd)] $0 $@" >> /tmp/disable-shared.log
+# shared build
+${PG_LINK:-emcc} -L${PREFIX}/lib -DPREFIX=/tmp/pglite -shared -sSIDE_MODULE=1 $@ -Wno-unused-function

pglite/other/bin/pg_config

@@ -0,0 +1,2 @@
+#!/bin/bash
+node /tmp/pglite/bin/pg_config.cjs $@

pglite/other/initdb

@@ -0,0 +1,3 @@
+#!/bin/bash
+. /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh
+TZ=UTC PGTZ=UTC node /tmp/pglite/bin/initdb.cjs $@

pglite/other/postgres

@@ -0,0 +1,3 @@
+#!/bin/bash
+. /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh
+TZ=UTC PGTZ=UTC PGDATA=${PGDATA} node /tmp/pglite/bin/postgres.cjs $@