From ec11b7bc6ef5d9eb2b61d1f237f0eeb4fe07167f Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Tue, 11 Mar 2025 14:47:16 +0000 Subject: [PATCH] HPCC-33405: Add support for ubuntu arm Bump vcpkg to 2025.02.13 with arm support Move antlr3c into vcpkg DC: Fix for arm64 register naming DC: Fix arm64 build issue in _cpyrevn DC: Move large zero array outside of functions (jlib file unittest) Rework dockerfiles/vcpkg/build.sh to support arm Signed-off-by Gordon Smith Move large zero array outside of functions --- .github/workflows/build-docker.yml | 25 +-- .github/workflows/build-vcpkg.yml | 25 +++ cmake_modules/vcpkg.cmake | 7 + dockerfiles/image.sh | 43 ++++- dockerfiles/vcpkg/build.sh | 166 ++++++++++-------- dockerfiles/vcpkg/centos-7.dockerfile | 4 - .../platform-core-ubuntu-22.04.dockerfile | 6 +- ecl/hql/hqlfold.cpp | 2 +- system/jlib/jmisc.hpp | 6 +- testing/unittests/filetests.cpp | 7 +- 10 files changed, 196 insertions(+), 95 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index e764b5d0be1..d76db2b46b7 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -58,6 +58,11 @@ on: description: 'Asset Name (if upload-package is true)' required: false default: 'build-docker-package' + tag_postfix: + type: string + description: 'Either -arm or empty string' + required: false + default: '' secrets: LNB_TOKEN: required: false @@ -65,7 +70,7 @@ on: jobs: build-docker: - runs-on: ubuntu-22.04 + runs-on: ubuntu-22.04${{ inputs.tag_postfix }} steps: - name: Free additional disk space (remove Android SDK + Tools) continue-on-error: true @@ -113,8 +118,8 @@ jobs: vcpkg_sha_short=$(git rev-parse --short=8 HEAD) echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT docker_build_label=hpccsystems/platform-build-${{ inputs.os }} - echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT - echo "docker_tag_candidate_base=$docker_build_label:$branch_label" >> $GITHUB_OUTPUT + echo "docker_tag=$docker_build_label:$vcpkg_sha_short${{ inputs.tag_postfix }}" >> $GITHUB_OUTPUT + echo "docker_tag_candidate_base=$docker_build_label:$branch_label${{ inputs.tag_postfix }}" >> $GITHUB_OUTPUT - name: Print vars shell: "bash" @@ -133,12 +138,12 @@ jobs: - uses: hendrikmuhs/ccache-action@v1.2 with: save: ${{ inputs.update-cache == true }} - key: docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} + key: docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} restore-keys: | - docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}- - docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}- - docker-${{ inputs.os }}-${{ inputs.build-type }}- - docker-${{ inputs.os }}- + docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}- + docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}- + docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}- + docker${{ inputs.tag_postfix }}-${{ inputs.os }}- - name: Docker build image uses: docker/build-push-action@v5 @@ -149,7 +154,7 @@ jobs: push: false load: true build-args: | - VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }} + VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }}${{ inputs.tag_postfix }} tags: | ${{ steps.vars.outputs.docker_tag }} ${{ steps.vars.outputs.docker_tag_candidate_base }} @@ -174,7 +179,7 @@ jobs: --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=delegated \ --mount source="${{ github.workspace }}/.ccache",target=/root/.ccache,type=bind,consistency=delegated \ ${{ steps.vars.outputs.docker_tag }} "\ - cmake -G Ninja -S /hpcc-dev/${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DVCPKG_FILES_DIR=/hpcc-dev -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} && \ + cmake -G Ninja -S /hpcc-dev/${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} && \ cmake --build /hpcc-dev/build --parallel ${{ inputs.upload-package == true && '--target package' || ''}}" done diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml index 0beea3e430e..690aa6d8aa0 100644 --- a/.github/workflows/build-vcpkg.yml +++ b/.github/workflows/build-vcpkg.yml @@ -64,6 +64,14 @@ jobs: asset-name: 'docker-package' secrets: inherit + build-docker-ubuntu-24_04-arm: + if: ${{ contains('pull_request,push', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: ubuntu-24.04 + tag_postfix: "-arm" + secrets: inherit + build-docker-ubuntu-24_04: if: ${{ contains('pull_request,push', github.event_name) }} uses: ./.github/workflows/build-docker.yml @@ -71,6 +79,14 @@ jobs: os: ubuntu-24.04 secrets: inherit + build-docker-ubuntu-22_04-arm: + if: ${{ contains('pull_request,push', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: ubuntu-22.04 + tag_postfix: "-arm" + secrets: inherit + build-docker-ubuntu-22_04: if: ${{ contains('pull_request,push', github.event_name) }} uses: ./.github/workflows/build-docker.yml @@ -214,3 +230,12 @@ jobs: build-type: 'Release' cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' secrets: inherit + + build-gh_runner-macos-14: + if: ${{ contains('pull_request,push', github.event_name) }} + uses: ./.github/workflows/build-gh_runner.yml + with: + os: macos-14 + build-type: 'Release' + cmake-configuration-ex: '-DCMAKE_OSX_ARCHITECTURES=arm64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + secrets: inherit diff --git a/cmake_modules/vcpkg.cmake b/cmake_modules/vcpkg.cmake index 9c0466ddedb..a35497ff9bd 100644 --- a/cmake_modules/vcpkg.cmake +++ b/cmake_modules/vcpkg.cmake @@ -19,8 +19,15 @@ elseif(APPLE) set(VCPKG_TARGET_TRIPLET "x64-osx" CACHE STRING "target triplet") endif() elseif(UNIX) + execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCHITECTURE) + string(STRIP ${ARCHITECTURE} ARCHITECTURE) + if(ARCHITECTURE MATCHES "arm" OR ARCHITECTURE MATCHES "aarch64") + set(VCPKG_HOST_TRIPLET "arm64-linux-dynamic" CACHE STRING "host triplet") + set(VCPKG_TARGET_TRIPLET "arm64-linux-dynamic" CACHE STRING "target triplet") + else() set(VCPKG_HOST_TRIPLET "x64-linux-dynamic" CACHE STRING "host triplet") set(VCPKG_TARGET_TRIPLET "x64-linux-dynamic" CACHE STRING "target triplet") + endif() endif() message("-- vcpkg settings:") diff --git a/dockerfiles/image.sh b/dockerfiles/image.sh index ecc9a34dd13..bf8fdf732f0 100755 --- a/dockerfiles/image.sh +++ b/dockerfiles/image.sh @@ -11,11 +11,14 @@ trapFunc() { trap 'trapFunc "${LINENO}/${BASH_LINENO}" "$?" "$BASH_COMMAND"' ERR globals() { + detect_arch SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )" ROOT_DIR=$(git rev-parse --show-toplevel) set +e - export $(grep -v '^#' $ROOT_DIR/.env | sed -e 's/\r$//' | xargs) > /dev/null + if [ -f $ROOT_DIR/.env ]; then + export $(grep -v '^#' $ROOT_DIR/.env | sed -e 's/\r$//' | xargs) > /dev/null + fi set -e GIT_REF=$(git rev-parse --short=8 HEAD) @@ -24,6 +27,10 @@ globals() { pushd $ROOT_DIR/vcpkg VCPKG_REF=$(git rev-parse --short=8 HEAD) popd + if [ "$ARCH" == "arm64" ]; then + VCPKG_REF="$VCPKG_REF-arm" + RELEASE_BASE_IMAGE="arm64v8/$RELEASE_BASE_IMAGE" + fi DOCKER_USERNAME="${DOCKER_USERNAME:-hpccbuilds}" CMAKE_OPTIONS="-G Ninja -DCPACK_THREADS=$(docker info --format '{{.NCPU}}') -DUSE_OPTIONAL=OFF -DCONTAINERIZED=ON -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON" @@ -265,6 +272,10 @@ build() { exit 1 fi + if [ "$ARCH" == "arm64" ]; then + base="$base" + fi + if [ "$RECONFIGURE" -eq 1 ]; then reconfigure fi @@ -331,6 +342,25 @@ function cleanup() { fi } +function detect_arch() { + if [ -z "$ARCH" ]; then + if [ "$(uname -m)" == "x86_64" ]; then + ARCH="x64" + elif [ "$(uname -m)" == "aarch64" ]; then + ARCH="arm64" + elif [ "$(uname -m)" == "arm64" ]; then + ARCH="arm64" + else + echo "Unsupported architecture: $(uname -m)" + exit 1 + fi + fi + if [ "$ARCH" != "x64" ] && [ "$ARCH" != "arm64" ]; then + echo "Unsupported architecture: $ARCH" + exit 1 + fi +} + trap cleanup EXIT status() { @@ -344,7 +374,9 @@ status() { echo "MODE: $MODE" echo "RECONFIGURE: $RECONFIGURE" echo "BUILD_OS: $BUILD_OS" + echo "RELEASE_BASE_IMAGE: $RELEASE_BASE_IMAGE" echo "HPCC_BUILD: $HPCC_BUILD" + echo "ARCH: $ARCH" } # Print usage information @@ -361,6 +393,7 @@ usage() { echo " -t, --tag tag the build volume with the current branch ref" echo " will preserve build state per branch" echo " -r, --reconfigure reconfigure CMake before building" + echo " -a, --architecture override default architecture (x64 or arm64)" } # Set default values @@ -369,8 +402,9 @@ MODE="release" RECONFIGURE=0 DEB_FILE="" BUILD_OS="ubuntu-22.04" -RELEASE_BASE_IMAGE="ubuntu:jammy-20230308" # Matches vcpkg base image (does not need to be an exact match) +RELEASE_BASE_IMAGE="ubuntu:22.04" # Matches vcpkg base image (does not need to be an exact match) TAG_BUILD=0 +ARCH="" # Parse command line arguments while [[ $# -gt 0 ]] @@ -413,6 +447,11 @@ case $key in RECONFIGURE=1 shift # past argument ;; + -a|--architecture) + ARCH="$2" + shift # past argument + shift # past value + ;; -h|--help) usage exit 0 diff --git a/dockerfiles/vcpkg/build.sh b/dockerfiles/vcpkg/build.sh index d98257658ad..d61f4f30470 100755 --- a/dockerfiles/vcpkg/build.sh +++ b/dockerfiles/vcpkg/build.sh @@ -1,101 +1,129 @@ #!/bin/bash set -e -SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"; - -export $(grep -v '^#' $SCRIPT_DIR/../../.env | xargs -d '\r' | xargs -d '\n') > /dev/null - -GITHUB_ACTOR="${GITHUB_ACTOR:-hpcc-systems}" -GITHUB_TOKEN="${GITHUB_TOKEN:-none}" -GITHUB_REF=$(git rev-parse --short=8 HEAD) -cd vcpkg -VCPKG_REF=$(git rev-parse --short=8 HEAD) -cd .. -GITHUB_BRANCH=$(git log -50 --pretty=format:"%D" | tr ',' '\n' | grep 'upstream/' | awk 'NR==1 {sub("upstream/", ""); print}' | xargs) -GITHUB_BRANCH=${GITHUB_BRANCH:-master} -DOCKER_USERNAME="${DOCKER_USERNAME:-hpccbuilds}" -DOCKER_PASSWORD="${DOCKER_PASSWORD:-none}" - -echo "SCRIPT_DIR: $SCRIPT_DIR" -echo "GITHUB_ACTOR: $GITHUB_ACTOR" -echo "GITHUB_TOKEN: $GITHUB_TOKEN" -echo "GITHUB_REF: $GITHUB_REF" -echo "VCPKG_REF: $VCPKG_REF" -echo "GITHUB_BRANCH: $GITHUB_BRANCH" -echo "DOCKER_USERNAME: $DOCKER_USERNAME" -echo "DOCKER_PASSWORD: $DOCKER_PASSWORD" - -docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD - -CMAKE_ALL_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON -DINCLUDE_PLUGINS=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DUSE_CPPUNIT=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON" -CMAKE_OPENBLAS_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF -DECLBLAS=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" -CMAKE_PLATFORM_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON -DPLATFORM=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" +function detect_arch() { + if [ -z "$ARCH" ]; then + if [ "$(uname -m)" == "x86_64" ]; then + ARCH="x64" + elif [ "$(uname -m)" == "aarch64" ]; then + ARCH="arm64" + elif [ "$(uname -m)" == "arm64" ]; then + ARCH="arm64" + else + echo "Unsupported architecture: $(uname -m)" + exit 1 + fi + fi + if [ "$ARCH" != "x64" ] && [ "$ARCH" != "arm64" ]; then + echo "Unsupported architecture: $ARCH" + exit 1 + fi +} + +function set_globals() { + detect_arch + SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"; + ROOT_DIR=$(git rev-parse --show-toplevel) + + set +e + if [ -f $ROOT_DIR/.env ]; then + export $(grep -v '^#' $ROOT_DIR/.env | sed -e 's/\r$//' | xargs) > /dev/null + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + fi + set -e + + GITHUB_ACTOR="${GITHUB_ACTOR:-hpcc-systems}" + GITHUB_TOKEN="${GITHUB_TOKEN:-none}" + GITHUB_REF=$(git rev-parse --short=8 HEAD) + pushd $ROOT_DIR/vcpkg + VCPKG_REF=$(git rev-parse --short=8 HEAD) + popd + if [ "$ARCH" == "arm64" ]; then + VCPKG_REF="$VCPKG_REF-arm" + fi + + GITHUB_BRANCH=$(git log -50 --pretty=format:"%D" | tr ',' '\n' | grep 'upstream/' | awk 'NR==1 {sub("upstream/", ""); print}' | xargs) + GITHUB_BRANCH=${GITHUB_BRANCH:-master} + DOCKER_USERNAME="${DOCKER_USERNAME:-hpccbuilds}" + DOCKER_PASSWORD="${DOCKER_PASSWORD:-none}" + + CMAKE_ALL_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON -DINCLUDE_PLUGINS=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DUSE_CPPUNIT=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON" + CMAKE_OPENBLAS_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF -DECLBLAS=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" + CMAKE_PLATFORM_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON -DPLATFORM=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" + + USER_ID=$(id -u) + GROUP_ID=$(id -g) +} + +function print_globals() { + echo "ARCH: $ARCH" + echo "ROOT_DIR: $ROOT_DIR" + echo "SCRIPT_DIR: $SCRIPT_DIR" + echo "GITHUB_REF: $GITHUB_REF" + echo "VCPKG_REF: $VCPKG_REF" + echo "GITHUB_BRANCH: $GITHUB_BRANCH" + echo "CMAKE_ALL_OPTIONS: $CMAKE_ALL_OPTIONS" + echo "CMAKE_OPENBLAS_OPTIONS: $CMAKE_OPENBLAS_OPTIONS" + echo "CMAKE_PLATFORM_OPTIONS: $CMAKE_PLATFORM_OPTIONS" + echo "GITHUB_ACTOR: $GITHUB_ACTOR" + echo "GITHUB_TOKEN: $GITHUB_TOKEN" + echo "DOCKER_USERNAME: $DOCKER_USERNAME" + echo "DOCKER_PASSWORD: $DOCKER_PASSWORD" + echo "USER_ID: $USER_ID" + echo "GROUP_ID: $GROUP_ID" +} + +function cleanup() { + rm -f $ROOT_DIR/vcpkg/vcpkg || true + kill $(jobs -p) +} function doBuild() { - # docker pull "hpccsystems/platform-build-base-$1:$VCPKG_REF" || true - # docker pull "hpccsystems/platform-build-$1:$VCPKG_REF" || true - # docker pull "hpccsystems/platform-build-$1:$GITHUB_BRANCH" || true docker buildx build --progress plain --rm -f "$SCRIPT_DIR/$1.dockerfile" \ - --build-arg DOCKER_NAMESPACE=$DOCKER_USERNAME \ --build-arg VCPKG_REF=$VCPKG_REF \ + --build-arg GROUP_ID=$GROUP_ID \ + --build-arg USER_ID=$USER_ID \ --cache-from hpccsystems/platform-build-$1:$VCPKG_REF \ --cache-from hpccsystems/platform-build-$1:$GITHUB_BRANCH \ -t hpccsystems/platform-build-$1:$VCPKG_REF \ -t hpccsystems/platform-build-$1:$GITHUB_BRANCH \ "$SCRIPT_DIR/." - # docker push hpccsystems/platform-build-$1:$VCPKG_REF - # docker push hpccsystems/platform-build-$1:$GITHUB_BRANCH + rm -f $ROOT_DIR/vcpkg/vcpkg - rm -f ./vcpkg/vcpkg - mkdir -p ./build-$1 + mkdir -p $ROOT_DIR/build/$1 CMAKE_OPTIONS_EXTRA="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" mkdir -p $HOME/.ccache docker run --rm \ + --user $USER_ID:$GROUP_ID \ --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached \ --mount source="$(realpath ~)/.cache/vcpkg",target=/root/.cache/vcpkg,type=bind,consistency=cached \ --mount source="$HOME/.ccache",target=/root/.ccache,type=bind,consistency=cached \ hpccsystems/platform-build-$1:$VCPKG_REF \ - "rm -rf /hpcc-dev/HPCC-Platform/build-$1/CMakeCache.txt /hpcc-dev/HPCC-Platform/build-$1/CMakeFiles && \ - cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_ALL_OPTIONS} ${CMAKE_OPTIONS_EXTRA} && \ - cmake --build /hpcc-dev/HPCC-Platform/build-$1 --parallel && \ + "rm -rf /hpcc-dev/HPCC-Platform/build/$1/CMakeCache.txt /hpcc-dev/HPCC-Platform/build/$1/CMakeFiles && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build/$1 ${CMAKE_ALL_OPTIONS} ${CMAKE_OPTIONS_EXTRA} && \ + cmake --build /hpcc-dev/HPCC-Platform/build/$1 --parallel || \ + cat /hpcc-dev/vcpkg_buildtrees/detect_compiler/config-x64-linux-dynamic-rel-err.log && \ echo 'Done'" + # docker run -it --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-ubuntu-22.04:latest bash - rm -f ./vcpkg/vcpkg - -# sudo chown -R $(id -u):$(id -g) ./build-$1 -# docker run -it --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-ubuntu-22.04:latest bash -} - -function cleanup() { - kill $(jobs -p) - rm -f ./vcpkg/vcpkg || true + rm -f $ROOT_DIR/vcpkg/vcpkg } -trap 'cleanup; exit' EXIT - -# ./vcpkg/bootstrap-vcpkg.sh -mkdir -p ./vcpkg-logs +set_globals +print_globals if [ "$1" != "" ]; then doBuild $1 else - doBuild ubuntu-24.04 &> vcpkg-logs/ubuntu-24.04.log & - doBuild ubuntu-22.04 &> vcpkg-logs/ubuntu-22.04.log & - doBuild ubuntu-20.04 &> vcpkg-logs/ubuntu-20.04.log & - doBuild rockylinux-8 &> vcpkg-logs/rockylinux-8.log & - doBuild centos-7 &> vcpkg-logs/centos-7.log & + trap 'cleanup; exit' EXIT + mkdir -p $ROOT_DIR/build/docker-logs + doBuild ubuntu-24.04 &> $ROOT_DIR/build/docker-logs/ubuntu-24.04.log & + doBuild ubuntu-22.04 &> $ROOT_DIR/build/docker-logs/ubuntu-22.04.log & + doBuild ubuntu-20.04 &> $ROOT_DIR/build/docker-logs/ubuntu-20.04.log & + doBuild rockylinux-8 &> $ROOT_DIR/build/docker-logs/rockylinux-8.log & + doBuild centos-7 &> $ROOT_DIR/build/docker-logs/centos-7.log & + wait fi - -wait - -# docker build --progress plain --pull --rm -f "$SCRIPT_DIR/core.dockerfile" \ -# -t $DOCKER_USERNAME/core:$GITHUB_REF \ -# -t $DOCKER_USERNAME/core:latest \ -# "build-ubuntu-22.04" -# docker push $DOCKER_USERNAME/core:$GITHUB_REF -# docker push $DOCKER_USERNAME/core:latest - -# docker run -it -d -p 8010:8010 core:latest touch /var/log/HPCCSystems/myesp/esp.log && /etc/init.d/hpcc-init start && tail -f /var/log/HPCCSystems/myesp/esp.log diff --git a/dockerfiles/vcpkg/centos-7.dockerfile b/dockerfiles/vcpkg/centos-7.dockerfile index f415699054e..19ab3186f89 100644 --- a/dockerfiles/vcpkg/centos-7.dockerfile +++ b/dockerfiles/vcpkg/centos-7.dockerfile @@ -7,8 +7,4 @@ RUN yum install -y \ ENTRYPOINT ["/bin/bash", "--login", "-c"] -RUN yum install -y \ - rpm-build && \ - yum -y clean all && rm -rf /var/cache - CMD ["/bin/bash"] diff --git a/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile index dc820de1e1a..17ac8c1ac0f 100644 --- a/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile +++ b/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile @@ -68,9 +68,9 @@ RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lf # Set the locale RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 RUN groupadd -g 10001 hpcc RUN useradd -s /bin/bash -m -r -N -c "hpcc runtime User" -u 10000 -g hpcc hpcc diff --git a/ecl/hql/hqlfold.cpp b/ecl/hql/hqlfold.cpp index e93da9a8f60..da19298f1e4 100644 --- a/ecl/hql/hqlfold.cpp +++ b/ecl/hql/hqlfold.cpp @@ -1355,7 +1355,7 @@ IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, const ch "add sp, sp, %[_poplen] \n\t" // Restore stack pointer (note, have already popped 8 registers, so poplen is len - 64) : "=r"(_int64result) : [_len] "r"(_len), [_poplen] "r"(_poplen), [strbuf] "r"(strbuf), [fh] "r"(_fh) - : "x2","x3","x4","x5","x6","x7","lr" // function we call may corrupt lr + : "x2","x3","x4","x5","x6","x7","x30" // function we call may corrupt x30 (also known as lr) ); int64result = _int64result; if (isRealvalue) diff --git a/system/jlib/jmisc.hpp b/system/jlib/jmisc.hpp index 4f75f5da4b9..fa01518a0c8 100644 --- a/system/jlib/jmisc.hpp +++ b/system/jlib/jmisc.hpp @@ -174,9 +174,9 @@ inline void _cpyrev8(void * _tgt, const void * _src) { *(unsigned __int64 *)_tgt = value; } inline void _cpyrevn(void * _tgt, const void * _src, unsigned len) { - char * tgt = (char *)_tgt; const char * src = (const char *)_src+len; - for (;len;len--) { - *tgt++ = *--src; + char * tgt = (char *)_tgt; const char * src = (const char *)_src; + for (unsigned i = 0; i < len; i++) { + tgt[i] = src[len - i - 1]; } } diff --git a/testing/unittests/filetests.cpp b/testing/unittests/filetests.cpp index c2c29ea718e..e15102fae16 100644 --- a/testing/unittests/filetests.cpp +++ b/testing/unittests/filetests.cpp @@ -43,6 +43,8 @@ #define CPPUNIT_ASSERT_EQUAL_STR(x, y) CPPUNIT_ASSERT_EQUAL(std::string(x ? x : ""),std::string(y ? y : "")) +static constexpr byte zeros[0x100000] = { 0 }; + class JlibFileTest : public CppUnit::TestFixture { public: @@ -135,12 +137,11 @@ class JlibFileTest : public CppUnit::TestFixture } void testCompressed() { - //patch the first block with zeros - constexpr byte zeros[0x100000] = { 0 }; - createCompressed(); readCompressed(false); + // patch the file with zeroes in various places, retest + write(0, sizeof(zeros), (void *)zeros); readCompressed(true);