feat: Bump vcpkg to 2025.02.13 with arm support

Move antlr3c into vcpkg DC: Fix for arm64 register naming Rework dockerfiles/vcpkg/build.sh to support arm Signed-off-by Gordon Smith <[email protected]>
hpcc-systems · Feb 27, 2025 · 0670a96 · 0670a96
1 parent 6c69419
commit 0670a96
Show file tree

Hide file tree

Showing 27 changed files with 433 additions and 192 deletions.
diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
@@ -58,14 +58,19 @@ on:
         description: 'Asset Name (if upload-package is true)'
         required: false
         default: 'build-docker-package'
+      tag_postfix:
+        type: string
+        description: 'Either -arm or empty string'
+        required: false
+        default: ''
     secrets:
       LNB_TOKEN:
         required: false
 
 jobs:
 
   build-docker:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-22.04${{ inputs.tag_postfix }}
     steps:
       - name: Free additional disk space (remove Android SDK + Tools)
         continue-on-error: true
@@ -112,7 +117,7 @@ jobs:
           vcpkg_sha_short=$(git rev-parse --short=8 HEAD)
           echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT
           docker_build_label=hpccsystems/platform-build-base-${{ inputs.os }}
-          echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT
+          echo "docker_tag=$docker_build_label:$vcpkg_sha_short${{ inputs.tag_postfix }}" >> $GITHUB_OUTPUT
 
       - name: Print vars
         shell: "bash"
@@ -128,12 +133,12 @@ jobs:
       - uses: hendrikmuhs/[email protected]
         with:
           save: ${{ inputs.update-cache == true }}
-          key: docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }}
+          key: docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }}
           restore-keys: |
-            docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-
-            docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-
-            docker-${{ inputs.os }}-${{ inputs.build-type }}-
-            docker-${{ inputs.os }}-
+            docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-
+            docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-
+            docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-
+            docker${{ inputs.tag_postfix }}-${{ inputs.os }}-
 
       - name: CMake Configure and Build
         shell: "bash"
@@ -152,7 +157,7 @@ jobs:
               --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=delegated \
               --mount source="${{ github.workspace }}/.ccache",target=/root/.ccache,type=bind,consistency=delegated \
               ${{ steps.vars.outputs.docker_tag }} "\
-              cmake -G Ninja -S /hpcc-dev/${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DVCPKG_FILES_DIR=/hpcc-dev -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} && \
+              cmake -G Ninja -S /hpcc-dev/${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} && \
               cmake --build /hpcc-dev/build --parallel ${{ inputs.upload-package == true && '--target package' || ''}}" 
           done
 

diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml
@@ -64,13 +64,29 @@ jobs:
       asset-name: 'docker-package'
     secrets: inherit
 
+  build-docker-ubuntu-24_04-arm:
+    if: ${{ contains('pull_request,push', github.event_name) }}
+    uses: ./.github/workflows/build-docker.yml
+    with:
+      os: ubuntu-24.04
+      tag_postfix: "-arm"
+    secrets: inherit
+
   build-docker-ubuntu-24_04:
     if: ${{ contains('pull_request,push', github.event_name) }}
     uses: ./.github/workflows/build-docker.yml
     with:
       os: ubuntu-24.04
     secrets: inherit
 
+  build-docker-ubuntu-22_04-arm:
+    if: ${{ contains('pull_request,push', github.event_name) }}
+    uses: ./.github/workflows/build-docker.yml
+    with:
+      os: ubuntu-22.04
+      tag_postfix: "-arm"
+    secrets: inherit
+
   build-docker-ubuntu-22_04:
     if: ${{ contains('pull_request,push', github.event_name) }}
     uses: ./.github/workflows/build-docker.yml
@@ -214,3 +230,12 @@ jobs:
       build-type: 'Release'
       cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
     secrets: inherit
+
+  build-gh_runner-macos-14:
+    if: ${{ contains('pull_request,push', github.event_name) }}
+    uses: ./.github/workflows/build-gh_runner.yml
+    with:
+      os: macos-14
+      build-type: 'Release'
+      cmake-configuration-ex: '-DCMAKE_OSX_ARCHITECTURES=arm64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
+    secrets: inherit
diff --git a/.gitmodules b/.gitmodules
@@ -1,9 +1,6 @@
 [submodule "esp/src/dgrid"]
 	path = esp/src/dgrid
 	url = https://github.com/hpcc-systems/dgrid.git
-[submodule "esp/services/ws_sql/libantlr3c"]
-	path = esp/services/ws_sql/libantlr3c
-	url = https://github.com/hpcc-systems/libantlr3c.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/hpcc-systems/vcpkg.git
diff --git a/cmake_modules/FindANTLR.cmake b/cmake_modules/FindANTLR.cmake
@@ -19,18 +19,24 @@
 # ANTLR_FOUND - ANTLR found in local system
 # ANTLR_BUILDTIME_JAR - The jar needed to build/generate ANTLR Lexers and Parsers
 # ANTLR_RUNTIME_JAR - The jar needed to build/generate ANTLR Lexers and Parsers
+# ANTLR_LIB - The runtime library needed by the generated ANTLR Lexers and Parsers
 ################################################################################
 
 include(UseJava)
 
-set(ANTLR_BUILDTIME_DEP "antlr-3.4-complete" CACHE STRING "ANTLR buildtime jar file name.")
-set(ANTLR_RUNTIME_DEP "antlr-runtime-3.4" CACHE STRING "ANTLR runtime jar file name.")
-set(ANTLR_PATH "${HPCC_SOURCE_DIR}/esp/services/ws_sql/libantlr3c" CACHE PATH "Location of ANTLR jar files.")
-set(ANTLR_PKG_FIND_ERROR_MSG "Could not locate jars.\nPlease run `git submodule update --init --recursive`\n")
+set(ANTLR_PKG_FIND_ERROR_MSG "Could not locate jars.\nPlease check vcpkg configure completed without issue.\n")
 
+find_library(ANTLR_LIB NAMES antlr3c
+    PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+)
 
-find_jar(ANTLR_BUILDTIME_JAR ${ANTLR_BUILDTIME_DEP} PATHS ${ANTLR_PATH})
-find_jar(ANTLR_RUNTIME_JAR ${ANTLR_RUNTIME_DEP} PATHS ${ANTLR_PATH})
+find_file(ANTLR_BUILDTIME_JAR "share/antlr3/antlr-3.4-complete.jar"
+    PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+)
+
+find_file(ANTLR_RUNTIME_JAR "share/antlr3/antlr-runtime-3.4.jar"
+    PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(

diff --git a/cmake_modules/plugins.cmake b/cmake_modules/plugins.cmake
@@ -85,6 +85,11 @@ if (USE_PARQUET)
 endif()
 
 #  vcpkg.json options  ---
+set(VCPKG_ANTLR3 "${VCPKG_SUPPRESS}")
+if (WSSQL_SERVICE)
+    set(VCPKG_ANTLR3 "${VCPKG_INCLUDE}")
+endif()
+
 set(VCPKG_APR "${VCPKG_SUPPRESS}")
 if (USE_APR)
     set(VCPKG_APR "${VCPKG_INCLUDE}")

diff --git a/cmake_modules/vcpkg.cmake b/cmake_modules/vcpkg.cmake
@@ -19,8 +19,15 @@ elseif(APPLE)
     set(VCPKG_TARGET_TRIPLET "x64-osx" CACHE STRING "target triplet")
   endif()
 elseif(UNIX)
+  execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCHITECTURE)
+  string(STRIP ${ARCHITECTURE} ARCHITECTURE)
+  if(ARCHITECTURE MATCHES "arm" OR ARCHITECTURE MATCHES "aarch64")
+    set(VCPKG_HOST_TRIPLET "arm64-linux-dynamic" CACHE STRING "host triplet")
+    set(VCPKG_TARGET_TRIPLET "arm64-linux-dynamic" CACHE STRING "target triplet")
+  else()
     set(VCPKG_HOST_TRIPLET "x64-linux-dynamic" CACHE STRING "host triplet")
     set(VCPKG_TARGET_TRIPLET "x64-linux-dynamic" CACHE STRING "target triplet")
+  endif()
 endif()
 
 message("-- vcpkg settings:")

diff --git a/dockerfiles/image.sh b/dockerfiles/image.sh
@@ -11,11 +11,14 @@ trapFunc() {
 trap 'trapFunc "${LINENO}/${BASH_LINENO}" "$?" "$BASH_COMMAND"' ERR
 
 globals() {
+    detect_arch
     SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"
     ROOT_DIR=$(git rev-parse --show-toplevel)
 
     set +e
-    export $(grep -v '^#' $ROOT_DIR/.env | sed -e 's/\r$//' | xargs) > /dev/null
+    if [ -f $ROOT_DIR/.env ]; then
+        export $(grep -v '^#' $ROOT_DIR/.env | sed -e 's/\r$//' | xargs) > /dev/null
+    fi
     set -e
 
     GIT_REF=$(git rev-parse --short=8 HEAD)
@@ -24,6 +27,10 @@ globals() {
     pushd $ROOT_DIR/vcpkg
     VCPKG_REF=$(git rev-parse --short=8 HEAD)
     popd
+    if [ "$ARCH" == "arm64" ]; then
+        VCPKG_REF="$VCPKG_REF-arm"
+        RELEASE_BASE_IMAGE="arm64v8/$RELEASE_BASE_IMAGE"
+    fi
     DOCKER_USERNAME="${DOCKER_USERNAME:-hpccbuilds}"
 
     CMAKE_OPTIONS="-G Ninja -DCPACK_THREADS=$(docker info --format '{{.NCPU}}') -DUSE_OPTIONAL=OFF -DCONTAINERIZED=ON -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON"
@@ -265,6 +272,10 @@ build() {
         exit 1
     fi
 
+    if [ "$ARCH" == "arm64" ]; then
+        base="$base"
+    fi
+
     if [ "$RECONFIGURE" -eq 1 ]; then
         reconfigure
     fi
@@ -331,6 +342,25 @@ function cleanup() {
     fi
 }
 
+function detect_arch() {
+    if [ -z "$ARCH" ]; then
+        if [ "$(uname -m)" == "x86_64" ]; then
+            ARCH="x64"
+        elif [ "$(uname -m)" == "aarch64" ]; then
+            ARCH="arm64"
+        elif [ "$(uname -m)" == "arm64" ]; then
+            ARCH="arm64"
+        else
+            echo "Unsupported architecture: $(uname -m)"
+            exit 1
+        fi
+    fi
+    if [ "$ARCH" != "x64" ] && [ "$ARCH" != "arm64" ]; then
+        echo "Unsupported architecture: $ARCH"
+        exit 1
+    fi
+}
+
 trap cleanup EXIT
 
 status() {
@@ -344,7 +374,9 @@ status() {
     echo "MODE: $MODE"
     echo "RECONFIGURE: $RECONFIGURE"
     echo "BUILD_OS: $BUILD_OS"
+    echo "RELEASE_BASE_IMAGE: $RELEASE_BASE_IMAGE"
     echo "HPCC_BUILD: $HPCC_BUILD"
+    echo "ARCH: $ARCH"
 }
 
 # Print usage information
@@ -361,6 +393,7 @@ usage() {
     echo "  -t, --tag               tag the build volume with the current branch ref"
     echo "                          will preserve build state per branch"
     echo "  -r, --reconfigure       reconfigure CMake before building"
+    echo "  -a, --architecture      override default architecture (x64 or arm64)"
 }
 
 # Set default values
@@ -369,8 +402,9 @@ MODE="release"
 RECONFIGURE=0
 DEB_FILE=""
 BUILD_OS="ubuntu-22.04"
-RELEASE_BASE_IMAGE="ubuntu:jammy-20230308" # Matches vcpkg base image (does not need to be an exact match)
+RELEASE_BASE_IMAGE="ubuntu:22.04" # Matches vcpkg base image (does not need to be an exact match)
 TAG_BUILD=0
+ARCH=""
 
 # Parse command line arguments
 while [[ $# -gt 0 ]]
@@ -413,6 +447,11 @@ case $key in
         RECONFIGURE=1
         shift # past argument
         ;;
+    -a|--architecture)
+        ARCH="$2"
+        shift # past argument
+        shift # past value
+        ;;
     -h|--help)
         usage
         exit 0

diff --git a/dockerfiles/vcpkg/arm64-platform-core-debug-ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/arm64-platform-core-debug-ubuntu-22.04.dockerfile
@@ -0,0 +1,28 @@
+##############################################################################
+#
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+##############################################################################
+
+# Create base container image to be used by all HPCC processes 
+
+ARG BASE_IMAGE=arm64-platform-core-ubuntu-22.04:latest
+FROM ${BASE_IMAGE} 
+
+USER root
+
+RUN apt-get install -y libcap2-bin
+RUN setcap cap_sys_ptrace+eip /usr/bin/gdb
+
+USER hpcc
diff --git a/dockerfiles/vcpkg/arm64-platform-core-ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/arm64-platform-core-ubuntu-22.04.dockerfile
@@ -0,0 +1,89 @@
+##############################################################################
+#
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+##############################################################################
+
+# Create base container image to be used by all HPCC processes 
+
+ARG BASE_IMAGE=ubuntu:22.04
+FROM ${BASE_IMAGE}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get clean -y && \
+    apt-get autoclean -y && \
+    apt-get install -y -f && \
+    apt-get autoremove -y && \
+    apt-get update -y && \
+    apt-get install --no-install-recommends -y \
+    default-jdk \
+    elfutils \
+    expect \
+    g++ \
+    git \
+    locales \
+    jq \
+    openssh-client \
+    openssh-server \
+    python3 \ 
+    python3-dev \ 
+    psmisc \
+    r-base-core \
+    r-cran-rcpp \
+    r-cran-inline \
+    rsync \
+    zip \
+    curl \
+    clang
+
+ARG USE_CPPUNIT=1
+RUN if [ ${USE_CPPUNIT} -eq 1 ] ; then apt-get install -y libcppunit-1.15-0 ; fi 
+
+# these are developer tools - we may want to move them elsewhere so that they are only in the incremental builds?
+
+RUN apt-get install -y \
+    dnsutils \
+    gdb \
+    nano 
+
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/bin/linux/arm64/kubectl && \
+    chmod +x ./kubectl && \
+    mv ./kubectl /usr/local/bin
+
+RUN curl -LO http://ports.ubuntu.com/pool/universe/g/git-lfs/git-lfs_3.0.2-1_arm64.deb && \
+    dpkg -i git-lfs_3.0.2-1_arm64.deb && \
+    rm git-lfs_3.0.2-1_arm64.deb
+
+# Set the locale
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8  
+ENV LANGUAGE en_US:en  
+ENV LC_ALL en_US.UTF-8     
+
+RUN groupadd -g 10001 hpcc
+RUN useradd -s /bin/bash -m -r -N -c "hpcc runtime User" -u 10000 -g hpcc hpcc
+RUN passwd -l hpcc 
+
+RUN mkdir /var/lib/HPCCSystems && chown hpcc:hpcc /var/lib/HPCCSystems
+RUN mkdir /var/log/HPCCSystems && chown hpcc:hpcc /var/log/HPCCSystems
+RUN mkdir /var/lock/HPCCSystems && chown hpcc:hpcc /var/lock/HPCCSystems
+RUN mkdir /var/run/HPCCSystems && chown hpcc:hpcc /var/run/HPCCSystems
+
+USER hpcc
+ENV PATH="/opt/HPCCSystems/bin:${PATH}"
+ENV HPCC_containerized=1
+WORKDIR /var/lib/HPCCSystems
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
diff --git a/dockerfiles/vcpkg/arm64-ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/arm64-ubuntu-22.04.dockerfile
@@ -0,0 +1,6 @@
+ARG VCPKG_REF=latest
+FROM hpccsystems/platform-build-base-ubuntu-22.04:hpcc-platform-9.10.x-arm
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]