From 6d68cb58079fb0e54a2daf9e967c0cf5db53ae4a Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 12 Mar 2025 13:17:10 -0700 Subject: [PATCH 01/50] Upgrade Protobuf C++ to 22.5 --- COMPILING.md | 4 +- buildscripts/grpc-java-artifacts/Dockerfile | 1 + .../Dockerfile.multiarch.base | 1 + .../Dockerfile.ubuntu2004.base | 1 + buildscripts/kokoro/android-interop.sh | 4 +- buildscripts/kokoro/android.sh | 4 +- buildscripts/kokoro/unix.sh | 4 +- buildscripts/make_dependencies.bat | 7 +-- buildscripts/make_dependencies.sh | 54 ++++++++++++------- buildscripts/toolchain.cmake | 9 ++++ compiler/build.gradle | 2 +- 11 files changed, 59 insertions(+), 32 deletions(-) create mode 100644 buildscripts/toolchain.cmake diff --git a/COMPILING.md b/COMPILING.md index de3cbb026c1..b7df1319beb 100644 --- a/COMPILING.md +++ b/COMPILING.md @@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code generation. Most users only need to use `skipCodegen=true` as discussed above. ### Build Protobuf -The codegen plugin is C++ code and requires protobuf 21.7 or later. +The codegen plugin is C++ code and requires protobuf 22.5 or later. For Linux, Mac and MinGW: ``` -$ PROTOBUF_VERSION=21.7 +$ PROTOBUF_VERSION=22.5 $ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz $ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz $ cd protobuf-$PROTOBUF_VERSION diff --git a/buildscripts/grpc-java-artifacts/Dockerfile b/buildscripts/grpc-java-artifacts/Dockerfile index 736babe9d8e..6df0a39bc47 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile +++ b/buildscripts/grpc-java-artifacts/Dockerfile @@ -3,6 +3,7 @@ FROM almalinux:8 RUN yum install -y \ autoconf \ automake \ + cmake \ diffutils \ gcc-c++ \ glibc-devel \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base index 8f7cfae2f52..f9d4fb4fcc3 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ + cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base index 2d11d76c373..126a417b58f 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ + cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/kokoro/android-interop.sh b/buildscripts/kokoro/android-interop.sh index b4adc8bed43..f987aea85f6 100755 --- a/buildscripts/kokoro/android-interop.sh +++ b/buildscripts/kokoro/android-interop.sh @@ -7,8 +7,8 @@ set -exu -o pipefail cd github/grpc-java -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS=-I/tmp/protobuf/include +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib export OS_NAME=$(uname) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 13983e747b7..08e564458a3 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -9,8 +9,8 @@ BASE_DIR="$(pwd)" cd "$BASE_DIR/github/grpc-java" -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS=-I/tmp/protobuf/include +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib export OS_NAME=$(uname) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index 1b88b56ab40..9d0cd5ac493 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -51,9 +51,9 @@ fi export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" # Make protobuf discoverable by :grpc-compiler +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS="-I/tmp/protobuf/include" ./gradlew grpc-compiler:clean $GRADLE_FLAGS diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 2bbfd394d46..64d475f8982 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,5 +1,6 @@ -set PROTOBUF_VER=21.7 -set CMAKE_NAME=cmake-3.3.2-win32-x86 +@rem set PROTOBUF_VER=21.7 +set PROTOBUF_VER=22.5 +set CMAKE_NAME=cmake-3.26.3-windows-x86_64 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" ( call :installProto || exit /b 1 @@ -20,7 +21,7 @@ if not exist "%CMAKE_NAME%" ( set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin :hasCmake @rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled -powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 +powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1 del protobuf.zip mkdir protobuf-%PROTOBUF_VER%\build diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 3d02a72f4eb..13164f9dbd4 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -3,13 +3,15 @@ # Build protoc set -evux -o pipefail -PROTOBUF_VERSION=21.7 +PROTOBUF_VERSION=22.5 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" DOWNLOAD_DIR=/tmp/source INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH" +BUILDSCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" mkdir -p $DOWNLOAD_DIR +cd "$DOWNLOAD_DIR" # Start with a sane default NUM_CPU=4 @@ -26,27 +28,39 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - if [[ ! -d "$DOWNLOAD_DIR"/protobuf-"${PROTOBUF_VERSION}" ]]; then - curl -Ls https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR + if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then + curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz fi - pushd $DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION} + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" + mkdir "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" + pushd "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" # install here so we don't need sudo if [[ "$ARCH" == x86* ]]; then - ./configure CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} --disable-shared \ - --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == aarch* ]]; then - ./configure --disable-shared --host=aarch64-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == ppc* ]]; then - ./configure --disable-shared --host=powerpc64le-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == s390* ]]; then - ./configure --disable-shared --host=s390x-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == loongarch* ]]; then - ./configure --disable-shared --host=loongarch64-unknown-linux-gnu --prefix="$INSTALL_DIR" + CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \ + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 + else + if [[ "$ARCH" == aarch_64 ]]; then + GCC_ARCH=aarch64-linux-gnu + elif [[ "$ARCH" == ppcle_64 ]]; then + GCC_ARCH=powerpc64le-linux-gnu + elif [[ "$ARCH" == s390_64 ]]; then + GCC_ARCH=s390x-linux-gnu + elif [[ "$ARCH" == loongarch_64 ]]; then + GCC_ARCH=loongarch64-unknown-linux-gnu + else + echo "Unknown architecture: $ARCH" + exit 1 + fi + cmake .. \ + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ + -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake fi - # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first - make clean - make V=0 -j$NUM_CPU - make install + cmake --build . -j "$NUM_CPU" + cmake --install . + [ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib" popd fi @@ -60,7 +74,7 @@ ln -s "$INSTALL_DIR" /tmp/protobuf cat < Date: Wed, 12 Mar 2025 20:15:51 -0700 Subject: [PATCH 02/50] Re-add downloading of absl --- buildscripts/make_dependencies.bat | 6 ++++++ buildscripts/make_dependencies.sh | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 64d475f8982..5c801fa56f9 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,5 +1,6 @@ @rem set PROTOBUF_VER=21.7 set PROTOBUF_VER=22.5 +set ABSL_VERSION=20230125.4 set CMAKE_NAME=cmake-3.26.3-windows-x86_64 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" ( @@ -24,6 +25,11 @@ set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1 del protobuf.zip +powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1 +powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1 +del absl.zip +rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp +rename abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp mkdir protobuf-%PROTOBUF_VER%\build pushd protobuf-%PROTOBUF_VER%\build diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 13164f9dbd4..32d8f8db69f 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -4,6 +4,7 @@ set -evux -o pipefail PROTOBUF_VERSION=22.5 +ABSL_VERSION=20230125.4 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" @@ -30,6 +31,9 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then else if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz + curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz + rmdir "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" + mv "abseil-cpp-$ABSL_VERSION" "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" fi # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" From b76908fa6ff65df0c254b88b0d2d9813aa1c4d44 Mon Sep 17 00:00:00 2001 From: deadEternally Date: Thu, 13 Mar 2025 21:46:01 +0530 Subject: [PATCH 03/50] Set LIBRARY_PATH to /tmp/protobuf/lib --- buildscripts/kokoro/unix.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index 9d0cd5ac493..c237b4ce041 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -54,6 +54,7 @@ export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib +export LIBRARY_PATH=/tmp/protobuf/lib ./gradlew grpc-compiler:clean $GRADLE_FLAGS From 76a613a963d281ea9b395690a2a6dae2936eacd0 Mon Sep 17 00:00:00 2001 From: deadEternally Date: Thu, 13 Mar 2025 22:02:49 +0530 Subject: [PATCH 04/50] Remove LD_LIBRARY_PATH to try fix "found unexpected dependencies" in the :grpc-compiler:checkArtifacts step --- buildscripts/kokoro/unix.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index c237b4ce041..e65825cac01 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -53,7 +53,6 @@ export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" # Make protobuf discoverable by :grpc-compiler export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" -export LD_LIBRARY_PATH=/tmp/protobuf/lib export LIBRARY_PATH=/tmp/protobuf/lib ./gradlew grpc-compiler:clean $GRADLE_FLAGS From 707a50a5d8c53774607e9f37742f8acfb9526e98 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 04:06:01 +0000 Subject: [PATCH 05/50] Add librt.so.1 to needed dependencies. --- compiler/check-artifact.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/check-artifact.sh b/compiler/check-artifact.sh index 4d0c2fa6286..768a67b763b 100755 --- a/compiler/check-artifact.sh +++ b/compiler/check-artifact.sh @@ -114,7 +114,7 @@ checkDependencies () white_list="KERNEL32\.dll\|msvcrt\.dll\|USER32\.dll" elif [[ "$OS" == linux ]]; then dump_cmd='objdump -x '"$1"' | grep "NEEDED"' - white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6" + white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6|librt\.so\.1" if [[ "$ARCH" == x86_32 ]]; then white_list="${white_list}\|libm\.so\.6" elif [[ "$ARCH" == x86_64 ]]; then From 948511329baf4d6bb7536ed1aebea1b038cf8653 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 05:52:18 +0000 Subject: [PATCH 06/50] Add libm.so.6 as an expected dependency for linux. --- buildscripts/make_dependencies.sh | 2 +- compiler/check-artifact.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 32d8f8db69f..17e1675d4f7 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -80,5 +80,5 @@ To compile with the build dependencies: export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" -export LD_LIBRARY_PATH=/tmp/protobuf/lib +export LIBRARY_PATH=/tmp/protobuf/lib EOF diff --git a/compiler/check-artifact.sh b/compiler/check-artifact.sh index 768a67b763b..12d7709a2a8 100755 --- a/compiler/check-artifact.sh +++ b/compiler/check-artifact.sh @@ -114,7 +114,7 @@ checkDependencies () white_list="KERNEL32\.dll\|msvcrt\.dll\|USER32\.dll" elif [[ "$OS" == linux ]]; then dump_cmd='objdump -x '"$1"' | grep "NEEDED"' - white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6|librt\.so\.1" + white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6\|librt\.so\.1\|libm\.so\.6" if [[ "$ARCH" == x86_32 ]]; then white_list="${white_list}\|libm\.so\.6" elif [[ "$ARCH" == x86_64 ]]; then From 5a3e3cc585f2450e5c58b8cee957de10a1b4a454 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 09:19:31 +0000 Subject: [PATCH 07/50] Replace parallel compilation option -j with env variable for the purpose since it doesn't work on multiarch build arch64. --- buildscripts/make_dependencies.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 17e1675d4f7..9fec0dad5b8 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -62,7 +62,8 @@ else -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake fi - cmake --build . -j "$NUM_CPU" + export CMAKE_BUILD_PARALLEL_LEVEL="$NUM_CPU" + cmake --build . cmake --install . [ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib" popd From 8d8e03c291f97965fec53c18400e503f0b32ed7a Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 13:30:01 +0000 Subject: [PATCH 08/50] On multiarch arch64, it requires cmake 3.13 to allow using abseil module instead of as a package, and this version is not available to install. Hence building cmake from source. Building the latest source of cmake needs openssl location to be specified (or have openssl turned off) so using the older version 3.13.4 --- buildscripts/grpc-java-artifacts/Dockerfile | 1 - .../grpc-java-artifacts/Dockerfile.multiarch.base | 1 - .../grpc-java-artifacts/Dockerfile.ubuntu2004.base | 1 - buildscripts/make_dependencies.sh | 14 +++++++++++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/buildscripts/grpc-java-artifacts/Dockerfile b/buildscripts/grpc-java-artifacts/Dockerfile index 6df0a39bc47..736babe9d8e 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile +++ b/buildscripts/grpc-java-artifacts/Dockerfile @@ -3,7 +3,6 @@ FROM almalinux:8 RUN yum install -y \ autoconf \ automake \ - cmake \ diffutils \ gcc-c++ \ glibc-devel \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base index f9d4fb4fcc3..8f7cfae2f52 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base @@ -4,7 +4,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ - cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base index 126a417b58f..2d11d76c373 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base @@ -4,7 +4,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ - cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 9fec0dad5b8..af9665e21db 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -5,6 +5,7 @@ set -evux -o pipefail PROTOBUF_VERSION=22.5 ABSL_VERSION=20230125.4 +CMAKE_VERSION=3.13.4 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" @@ -29,6 +30,16 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else + if [[ ! -d "cmake-${CMAKE_VERSION}" ]]; then + curl -Ls "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" | tar xz + fi + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + rm -rf "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}/bin" + cd "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}" + ./bootstrap + make + make install + ln -s /usr/local/bin/cmake /usr/bin/cmake if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz @@ -60,7 +71,8 @@ else cmake .. \ -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ - -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake + -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake \ + -B. fi export CMAKE_BUILD_PARALLEL_LEVEL="$NUM_CPU" cmake --build . From a5ec1ae427e208add36696dfca74e7ec0374cbdf Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 14:00:01 +0000 Subject: [PATCH 09/50] Fix mistake in current directory used. --- buildscripts/make_dependencies.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index af9665e21db..1bbcde5cfa9 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -40,6 +40,7 @@ else make make install ln -s /usr/local/bin/cmake /usr/bin/cmake + cd "$DOWNLOAD_DIR" if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz From 54c962ae7bc26e339e73d33be078127ad1e0e132 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 19 Mar 2025 12:44:01 +0000 Subject: [PATCH 10/50] Fix missing software deps --- buildscripts/grpc-java-artifacts/Dockerfile | 5 ++++- .../grpc-java-artifacts/Dockerfile.multiarch.base | 10 ++++++++++ .../Dockerfile.ubuntu2004.base | 9 +++++++++ buildscripts/make_dependencies.sh | 15 +++------------ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/buildscripts/grpc-java-artifacts/Dockerfile b/buildscripts/grpc-java-artifacts/Dockerfile index 736babe9d8e..e43e2c0bf3d 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile +++ b/buildscripts/grpc-java-artifacts/Dockerfile @@ -27,7 +27,10 @@ RUN mkdir -p "$ANDROID_HOME/cmdline-tools" && \ mv "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest" && \ yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + # Install Maven RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ tar xz -C /var/local -ENV PATH /var/local/apache-maven-3.8.8/bin:$PATH +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base index 8f7cfae2f52..27c3f9a154c 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base @@ -10,5 +10,15 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-aarch64-linux-gnu \ g++-powerpc64le-linux-gnu \ openjdk-8-jdk \ + pkg-config \ && \ rm -rf /var/lib/apt/lists/* + +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + +# Install Maven +RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ + tar xz -C /var/local +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH + diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base index 2d11d76c373..40999502b2c 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base @@ -9,5 +9,14 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ curl \ g++-s390x-linux-gnu \ openjdk-8-jdk \ + pkg-config \ && \ rm -rf /var/lib/apt/lists/* + +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + +# Install Maven +RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ + tar xz -C /var/local +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 1bbcde5cfa9..e3abb0f39cc 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -5,7 +5,7 @@ set -evux -o pipefail PROTOBUF_VERSION=22.5 ABSL_VERSION=20230125.4 -CMAKE_VERSION=3.13.4 +CMAKE_VERSION=3.26.3 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" @@ -30,16 +30,6 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - if [[ ! -d "cmake-${CMAKE_VERSION}" ]]; then - curl -Ls "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" | tar xz - fi - # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first - rm -rf "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}/bin" - cd "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}" - ./bootstrap - make - make install - ln -s /usr/local/bin/cmake /usr/bin/cmake cd "$DOWNLOAD_DIR" if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz @@ -55,7 +45,8 @@ else if [[ "$ARCH" == x86* ]]; then CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \ -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 \ + -B. else if [[ "$ARCH" == aarch_64 ]]; then GCC_ARCH=aarch64-linux-gnu From 097c01ae25d04ec4785cd49471b21a2a13efb40b Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 30 Apr 2025 16:02:09 +0000 Subject: [PATCH 11/50] Windows changes. --- buildscripts/kokoro/windows32.bat | 37 +- buildscripts/kokoro/windows64.bat | 99 +++-- compiler/build.gradle | 649 +++++++++++++++--------------- 3 files changed, 422 insertions(+), 363 deletions(-) diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index ffd4d3b99a6..d14556e425a 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -15,20 +15,22 @@ set ESCWORKSPACE=%WORKSPACE:\=\\% @rem Clear JAVA_HOME to prevent a different Java version from being used set JAVA_HOME= -set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH% mkdir grpc-java-helper32 cd grpc-java-helper32 -call "%VS140COMNTOOLS%\vsvars32.bat" || exit /b 1 +call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars32.bat" || exit /b 1 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 cd "%WORKSPACE%" SET TARGET_ARCH=x86_32 SET FAIL_ON_WARNINGS=true -SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\Release -SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\include -SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true +SET PROTOBUF_VER=22.5 +SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig +SET VC_PROTOBUF_LIB_PATHS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib +SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include +call :Get_Libs +SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% build" @@ -50,3 +52,28 @@ IF NOT %GRADLEEXIT% == 0 ( cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:publish" || exit /b 1 + +goto :eof +:Get_Libs +SetLocal EnableDelayedExpansion +set "libs_list=" +for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( + for %%b in (%%a) do ( + set lib=%%b + set libfirst2char=!lib:~0,2! + if !libfirst2char!==-l ( + @rem remove the leading -l + set lib=!lib:~2! + @rem remove spaces + set lib=!lib: =! + @rem Because protobuf is specified as libprotobuf and elsewhere + if !lib! NEQ protobuf ( + set lib=!lib!.lib, + set libs_list=!libs_list! !lib! + ) + ) + ) +) +EndLocal & set "VC_PROTOBUF_LIBS=%libs_list%" +exit /b 0 + diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 8542f1c0536..7b91fca0a5f 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -1,36 +1,63 @@ -@rem ########################################################################## -@rem -@rem Builds artifacts for x86_64 into %WORKSPACE%\artifacts\ -@rem -@rem ########################################################################## - -type c:\VERSION - -@rem Enter repo root -cd /d %~dp0\..\.. - -set WORKSPACE=T:\src\github\grpc-java -set ESCWORKSPACE=%WORKSPACE:\=\\% - -@rem Clear JAVA_HOME to prevent a different Java version from being used -set JAVA_HOME= -set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH% - -mkdir grpc-java-helper64 -cd grpc-java-helper64 -call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" || exit /b 1 -call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 - -cd "%WORKSPACE%" - -SET TARGET_ARCH=x86_64 -SET FAIL_ON_WARNINGS=true -SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\Release -SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\include -SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true -SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" - -@rem make sure no daemons have any files open -cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" - -cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts grpc-compiler:clean grpc-compiler:build grpc-compiler:publish" || exit /b 1 +@rem ########################################################################## +@rem +@rem Builds artifacts for x86_64 into %WORKSPACE%\artifacts\ +@rem +@rem ########################################################################## + +type c:\VERSION + +@rem Enter repo root +cd /d %~dp0\..\.. + +set WORKSPACE=T:\src\github\grpc-java +set ESCWORKSPACE=%WORKSPACE:\=\\% + +@rem Clear JAVA_HOME to prevent a different Java version from being used +set JAVA_HOME= + +mkdir grpc-java-helper64 +cd grpc-java-helper64 +call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1 +call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 + +cd "%WORKSPACE%" + +SET TARGET_ARCH=x86_64 +SET FAIL_ON_WARNINGS=true +SET PROTOBUF_VER=22.5 +SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig +SET VC_PROTOBUF_LIB_PATHS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib +SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include +call :Get_Libs +SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true +SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" + +@rem make sure no daemons have any files open +cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" + +cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts grpc-compiler:clean grpc-compiler:build grpc-compiler:publish" || exit /b 1 + +goto :eof +:Get_Libs +SetLocal EnableDelayedExpansion +set "libs_list=" +for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( + for %%b in (%%a) do ( + set lib=%%b + set libfirst2char=!lib:~0,2! + if !libfirst2char!==-l ( + @rem remove the leading -l + set lib=!lib:~2! + @rem remove spaces + set lib=!lib: =! + @rem Because protobuf is specified as libprotobuf and elsewhere + if !lib! NEQ protobuf ( + set lib=!lib!.lib, + set libs_list=!libs_list! !lib! + ) + ) + ) +) +EndLocal & set "VC_PROTOBUF_LIBS=%libs_list%" +exit /b 0 + diff --git a/compiler/build.gradle b/compiler/build.gradle index 53e0fd641e3..1bcfe1e5445 100644 --- a/compiler/build.gradle +++ b/compiler/build.gradle @@ -1,322 +1,327 @@ -plugins { - id "cpp" - id "java" - id "maven-publish" - - id "com.google.protobuf" -} - -description = 'The protoc plugin for gRPC Java' - -def artifactStagingPath = "$buildDir/artifacts" as File -// Adds space-delimited arguments from the environment variable env to the -// argList. -def addEnvArgs = { env, argList -> - def value = System.getenv(env) - if (value != null) { - value.split(' +').each() { it -> argList.add(it) } - } -} - -// Adds corresponding "-l" option to the argList if libName is not found in -// LDFLAGS. This is only used for Mac because when building for uploadArchives -// artifacts, we add the ".a" files directly to LDFLAGS and without "-l" in -// order to get statically linked, otherwise we add the libraries through "-l" -// so that they can be searched for in default search paths. -def addLibraryIfNotLinked = { libName, argList -> - def ldflags = System.env.LDFLAGS - if (ldflags == null || !ldflags.contains('lib' + libName + '.a')) { - argList.add('-l' + libName) - } -} - -def String arch = rootProject.hasProperty('targetArch') ? rootProject.targetArch : osdetector.arch -def boolean vcDisable = rootProject.hasProperty('vcDisable') ? rootProject.vcDisable : false -def boolean usingVisualCpp // Whether VisualCpp is actually available and selected - -model { - toolChains { - // If you have both VC and Gcc installed, VC will be selected, unless you - // set 'vcDisable=true' - if (!vcDisable) { - visualCpp(VisualCpp) { - // Prefer vcvars-provided environment over registry-discovered environment - def String vsDir = System.getenv("VSINSTALLDIR") - def String winDir = System.getenv("WindowsSdkDir") - if (vsDir != null && winDir != null) { - installDir = vsDir - windowsSdkDir = winDir - } - } - } - gcc(Gcc) { - target("ppcle_64") { - cppCompiler.executable = 'powerpc64le-linux-gnu-g++' - linker.executable = 'powerpc64le-linux-gnu-g++' - } - target("aarch_64") { - cppCompiler.executable = 'aarch64-linux-gnu-g++' - linker.executable = 'aarch64-linux-gnu-g++' - } - target("s390_64") { - cppCompiler.executable = 's390x-linux-gnu-g++' - linker.executable = 's390x-linux-gnu-g++' - } - target("loongarch_64") - } - clang(Clang) { - target("aarch_64") {} - } - } - - platforms { - x86_32 { architecture "x86" } - x86_64 { architecture "x86_64" } - ppcle_64 { architecture "ppcle_64" } - aarch_64 { architecture "aarch_64" } - s390_64 { architecture "s390_64" } - loongarch_64 { architecture "loongarch_64" } - } - - components { - java_plugin(NativeExecutableSpec) { - if (arch in [ - 'x86_32', - 'x86_64', - 'ppcle_64', - 'aarch_64', - 's390_64', - 'loongarch_64' - ]) { - // If arch is not within the defined platforms, we do not specify the - // targetPlatform so that Gradle will choose what is appropriate. - targetPlatform arch - } - baseName "$protocPluginBaseName" - } - } - - binaries { - all { - if (toolChain in Gcc || toolChain in Clang) { - cppCompiler.define("GRPC_VERSION", version) - cppCompiler.args "--std=c++14" - addEnvArgs("CXXFLAGS", cppCompiler.args) - addEnvArgs("CPPFLAGS", cppCompiler.args) - if (osdetector.os == "osx") { - cppCompiler.args "-mmacosx-version-min=10.7", "-stdlib=libc++" - addLibraryIfNotLinked('protoc', linker.args) - addLibraryIfNotLinked('protobuf', linker.args) - } else if (osdetector.os == "windows") { - linker.args "-static", "-lprotoc", "-lprotobuf", "-static-libgcc", "-static-libstdc++", - "-s" - } else if (osdetector.arch == "ppcle_64") { - linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-Wl,-Bdynamic", "-lpthread", "-s" - } else { - // Link protoc, protobuf, libgcc and libstdc++ statically. - // Link other (system) libraries dynamically. - // Clang under OSX doesn't support these options. - linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-static-libgcc", - "-Wl,-Bdynamic", "-lpthread", "-s" - } - addEnvArgs("LDFLAGS", linker.args) - } else if (toolChain in VisualCpp) { - usingVisualCpp = true - cppCompiler.define("GRPC_VERSION", version) - cppCompiler.args "/EHsc", "/MT" - if (rootProject.hasProperty('vcProtobufInclude')) { - cppCompiler.args "/I${rootProject.vcProtobufInclude}" - } - linker.args "libprotobuf.lib", "libprotoc.lib" - if (rootProject.hasProperty('vcProtobufLibs')) { - linker.args "/LIBPATH:${rootProject.vcProtobufLibs}" - } - } - } - } -} - -sourceSets { - testLite { - proto { setSrcDirs(['src/test/proto']) } - } -} - -dependencies { - testImplementation project(':grpc-protobuf'), - project(':grpc-stub'), - libraries.javax.annotation - testLiteImplementation project(':grpc-protobuf-lite'), - project(':grpc-stub'), - libraries.javax.annotation -} - -tasks.named("compileTestJava").configure { - options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" -} - -tasks.named("compileTestLiteJava").configure { - options.compilerArgs = compileTestJava.options.compilerArgs - options.compilerArgs += [ - "-Xlint:-cast" - ] - options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" -} - -tasks.named("checkstyleTestLite").configure { - enabled = false -} - -protobuf { - protoc { - if (project.hasProperty('protoc')) { - path = project.protoc - } else { - artifact = libs.protobuf.protoc.get() - } - } - plugins { - grpc { path = javaPluginPath } - } - generateProtoTasks { - all().configureEach { - dependsOn 'java_pluginExecutable' - inputs.file javaPluginPath - } - ofSourceSet('test').configureEach { - plugins { grpc {} } - } - ofSourceSet('testLite').configureEach { - builtins { - java { option 'lite' } - } - plugins { - grpc { - option 'lite' - option '@generated=omit' - } - } - } - } -} - -println "*** Building codegen requires Protobuf" -println "*** Please refer to https://github.com/grpc/grpc-java/blob/master/COMPILING.md#how-to-build-code-generation-plugin" - -tasks.register("buildArtifacts", Copy) { - dependsOn 'java_pluginExecutable' - from("$buildDir/exe") { - if (osdetector.os != 'windows') { - rename 'protoc-gen-grpc-java', '$0.exe' - } - } - into artifactStagingPath -} - -base { - archivesName = "$protocPluginBaseName" -} - -def checkArtifacts = tasks.register("checkArtifacts") { - dependsOn buildArtifacts - doLast { - if (!usingVisualCpp) { - def ret = exec { - executable 'bash' - args 'check-artifact.sh', osdetector.os, arch - } - if (ret.exitValue != 0) { - throw new GradleException("check-artifact.sh exited with " + ret.exitValue) - } - } else { - def exeName = "$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" - def os = new ByteArrayOutputStream() - def ret = exec { - executable 'dumpbin' - args '/nologo', '/dependents', exeName - standardOutput = os - } - if (ret.exitValue != 0) { - throw new GradleException("dumpbin exited with " + ret.exitValue) - } - def dlls = os.toString() =~ /Image has the following dependencies:\s+(.*)\s+Summary/ - if (dlls[0][1] != "KERNEL32.dll") { - throw new Exception("unexpected dll deps: " + dlls[0][1]); - } - os.reset() - ret = exec { - executable 'dumpbin' - args '/nologo', '/headers', exeName - standardOutput = os - } - if (ret.exitValue != 0) { - throw new GradleException("dumpbin exited with " + ret.exitValue) - } - def machine = os.toString() =~ / machine \(([^)]+)\)/ - def expectedArch = [x86_32: "x86", x86_64: "x64"][arch] - if (machine[0][1] != expectedArch) { - throw new Exception("unexpected architecture: " + machine[0][1]); - } - } - } -} - -// Exe files are skipped by Maven by default. Override it. -// Also skip jar files that is generated by the java plugin. -publishing { - publications { - maven(MavenPublication) { - // Removes all artifacts since grpc-compiler doesn't generates any Jar - artifacts = [] - artifactId 'protoc-gen-grpc-java' - artifact("$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" as File) { - classifier osdetector.os + "-" + arch - extension "exe" - builtBy checkArtifacts - } - pom.withXml { - // This isn't any sort of Java archive artifact, and OSSRH doesn't enforce - // javadoc for 'pom' packages. 'exe' would be a more appropriate packaging - // value, but it isn't clear how that will be interpreted. In addition, - // 'pom' is typically the value used when building an exe with Maven. - asNode().project.packaging*.value = 'pom' - } - } - } -} - -def configureTestTask(Task task, String dep, String serviceName) { - def genDir = files(tasks.named("generateTest${dep}Proto")).singleFile - def genFile = "${genDir}/grpc/io/grpc/testing/compiler/${serviceName}Grpc.java" - task.dependsOn "generateTest${dep}Proto" - task.inputs.file genFile - if (osdetector.os != 'windows') { - task.executable "diff" - task.args "-u" - } else { - task.executable "fc" - } - task.args layout.projectDirectory.file("src/test${dep}/golden/${serviceName}.java.txt") - task.args genFile - // Register an output to allow up-to-date checking - task.outputs.file(layout.buildDirectory.file(task.name)) -} - -def testGolden = tasks.register("testGolden", Exec) { - configureTestTask(it, '', 'TestService') -} -def testLiteGolden = tasks.register("testLiteGolden", Exec) { - configureTestTask(it, 'Lite', 'TestService') -} -def testDeprecatedGolden = tasks.register("testDeprecatedGolden", Exec) { - configureTestTask(it, '', 'TestDeprecatedService') -} -def testDeprecatedLiteGolden = tasks.register("testDeprecatedLiteGolden", Exec) { - configureTestTask(it, 'Lite', 'TestDeprecatedService') -} -tasks.named("test").configure { - dependsOn testGolden - dependsOn testLiteGolden - dependsOn testDeprecatedGolden - dependsOn testDeprecatedLiteGolden -} +plugins { + id "cpp" + id "java" + id "maven-publish" + + id "com.google.protobuf" +} + +description = 'The protoc plugin for gRPC Java' + +def artifactStagingPath = "$buildDir/artifacts" as File +// Adds space-delimited arguments from the environment variable env to the +// argList. +def addEnvArgs = { env, argList -> + def value = System.getenv(env) + if (value != null) { + value.split(' +').each() { it -> argList.add(it) } + } +} + +// Adds corresponding "-l" option to the argList if libName is not found in +// LDFLAGS. This is only used for Mac because when building for uploadArchives +// artifacts, we add the ".a" files directly to LDFLAGS and without "-l" in +// order to get statically linked, otherwise we add the libraries through "-l" +// so that they can be searched for in default search paths. +def addLibraryIfNotLinked = { libName, argList -> + def ldflags = System.env.LDFLAGS + if (ldflags == null || !ldflags.contains('lib' + libName + '.a')) { + argList.add('-l' + libName) + } +} + +def String arch = rootProject.hasProperty('targetArch') ? rootProject.targetArch : osdetector.arch +def boolean vcDisable = rootProject.hasProperty('vcDisable') ? rootProject.vcDisable : false +def boolean usingVisualCpp // Whether VisualCpp is actually available and selected + +model { + toolChains { + // If you have both VC and Gcc installed, VC will be selected, unless you + // set 'vcDisable=true' + if (!vcDisable) { + visualCpp(VisualCpp) { + // Prefer vcvars-provided environment over registry-discovered environment + def String vsDir = System.getenv("VSINSTALLDIR") + def String winDir = System.getenv("WindowsSdkDir") + if (vsDir != null && winDir != null) { + installDir = vsDir + windowsSdkDir = winDir + } + } + } + gcc(Gcc) { + target("ppcle_64") { + cppCompiler.executable = 'powerpc64le-linux-gnu-g++' + linker.executable = 'powerpc64le-linux-gnu-g++' + } + target("aarch_64") { + cppCompiler.executable = 'aarch64-linux-gnu-g++' + linker.executable = 'aarch64-linux-gnu-g++' + } + target("s390_64") { + cppCompiler.executable = 's390x-linux-gnu-g++' + linker.executable = 's390x-linux-gnu-g++' + } + target("loongarch_64") + } + clang(Clang) { + target("aarch_64") {} + } + } + + platforms { + x86_32 { architecture "x86" } + x86_64 { architecture "x86_64" } + ppcle_64 { architecture "ppcle_64" } + aarch_64 { architecture "aarch_64" } + s390_64 { architecture "s390_64" } + loongarch_64 { architecture "loongarch_64" } + } + + components { + java_plugin(NativeExecutableSpec) { + if (arch in [ + 'x86_32', + 'x86_64', + 'ppcle_64', + 'aarch_64', + 's390_64', + 'loongarch_64' + ]) { + // If arch is not within the defined platforms, we do not specify the + // targetPlatform so that Gradle will choose what is appropriate. + targetPlatform arch + } + baseName "$protocPluginBaseName" + } + } + + binaries { + all { + if (toolChain in Gcc || toolChain in Clang) { + cppCompiler.define("GRPC_VERSION", version) + cppCompiler.args "--std=c++0x" + addEnvArgs("CXXFLAGS", cppCompiler.args) + addEnvArgs("CPPFLAGS", cppCompiler.args) + if (osdetector.os == "osx") { + cppCompiler.args "-mmacosx-version-min=10.7", "-stdlib=libc++" + addLibraryIfNotLinked('protoc', linker.args) + addLibraryIfNotLinked('protobuf', linker.args) + } else if (osdetector.os == "windows") { + linker.args "-static", "-lprotoc", "-lprotobuf", "-static-libgcc", "-static-libstdc++", + "-s" + } else if (osdetector.arch == "ppcle_64") { + linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-Wl,-Bdynamic", "-lpthread", "-s" + } else { + // Link protoc, protobuf, libgcc and libstdc++ statically. + // Link other (system) libraries dynamically. + // Clang under OSX doesn't support these options. + linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-static-libgcc", + "-Wl,-Bdynamic", "-lpthread", "-s" + } + addEnvArgs("LDFLAGS", linker.args) + } else if (toolChain in VisualCpp) { + usingVisualCpp = true + cppCompiler.define("GRPC_VERSION", version) + cppCompiler.args "/EHsc", "/MT" + if (rootProject.hasProperty('vcProtobufInclude')) { + cppCompiler.args "/I${rootProject.vcProtobufInclude}" + } + if (rootProject.hasProperty('vcProtobufLibs')) { + String libsList = rootProject.property('vcProtobufLibs') as String + linker.args.add("libprotoc.lib") + linker.args.add("libprotobuf.lib") + libsList.split(',').each() { lib -> linker.args.add(lib) } + } + if (rootProject.hasProperty('vcProtobufLibPaths')) { + linker.args "/LIBPATH:${rootProject.vcProtobufLibPaths}" + } + } + } + } +} + +sourceSets { + testLite { + proto { setSrcDirs(['src/test/proto']) } + } +} + +dependencies { + testImplementation project(':grpc-protobuf'), + project(':grpc-stub'), + libraries.javax.annotation + testLiteImplementation project(':grpc-protobuf-lite'), + project(':grpc-stub'), + libraries.javax.annotation +} + +tasks.named("compileTestJava").configure { + options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" +} + +tasks.named("compileTestLiteJava").configure { + options.compilerArgs = compileTestJava.options.compilerArgs + options.compilerArgs += [ + "-Xlint:-cast" + ] + options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" +} + +tasks.named("checkstyleTestLite").configure { + enabled = false +} + +protobuf { + protoc { + if (project.hasProperty('protoc')) { + path = project.protoc + } else { + artifact = libs.protobuf.protoc.get() + } + } + plugins { + grpc { path = javaPluginPath } + } + generateProtoTasks { + all().configureEach { + dependsOn 'java_pluginExecutable' + inputs.file javaPluginPath + } + ofSourceSet('test').configureEach { + plugins { grpc {} } + } + ofSourceSet('testLite').configureEach { + builtins { + java { option 'lite' } + } + plugins { + grpc { + option 'lite' + option '@generated=omit' + } + } + } + } +} + +println "*** Building codegen requires Protobuf" +println "*** Please refer to https://github.com/grpc/grpc-java/blob/master/COMPILING.md#how-to-build-code-generation-plugin" + +tasks.register("buildArtifacts", Copy) { + dependsOn 'java_pluginExecutable' + from("$buildDir/exe") { + if (osdetector.os != 'windows') { + rename 'protoc-gen-grpc-java', '$0.exe' + } + } + into artifactStagingPath +} + +base { + archivesName = "$protocPluginBaseName" +} + +def checkArtifacts = tasks.register("checkArtifacts") { + dependsOn buildArtifacts + doLast { + if (!usingVisualCpp) { + def ret = exec { + executable 'bash' + args 'check-artifact.sh', osdetector.os, arch + } + if (ret.exitValue != 0) { + throw new GradleException("check-artifact.sh exited with " + ret.exitValue) + } + } else { + def exeName = "$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" + def os = new ByteArrayOutputStream() + def ret = exec { + executable 'dumpbin' + args '/nologo', '/dependents', exeName + standardOutput = os + } + if (ret.exitValue != 0) { + throw new GradleException("dumpbin exited with " + ret.exitValue) + } + def dlls = os.toString() =~ /Image has the following dependencies:\s+(.*)\s+Summary/ + if (dlls[0][1] != "KERNEL32.dll") { + throw new Exception("unexpected dll deps: " + dlls[0][1]); + } + os.reset() + ret = exec { + executable 'dumpbin' + args '/nologo', '/headers', exeName + standardOutput = os + } + if (ret.exitValue != 0) { + throw new GradleException("dumpbin exited with " + ret.exitValue) + } + def machine = os.toString() =~ / machine \(([^)]+)\)/ + def expectedArch = [x86_32: "x86", x86_64: "x64"][arch] + if (machine[0][1] != expectedArch) { + throw new Exception("unexpected architecture: " + machine[0][1]); + } + } + } +} + +// Exe files are skipped by Maven by default. Override it. +// Also skip jar files that is generated by the java plugin. +publishing { + publications { + maven(MavenPublication) { + // Removes all artifacts since grpc-compiler doesn't generates any Jar + artifacts = [] + artifactId 'protoc-gen-grpc-java' + artifact("$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" as File) { + classifier osdetector.os + "-" + arch + extension "exe" + builtBy checkArtifacts + } + pom.withXml { + // This isn't any sort of Java archive artifact, and OSSRH doesn't enforce + // javadoc for 'pom' packages. 'exe' would be a more appropriate packaging + // value, but it isn't clear how that will be interpreted. In addition, + // 'pom' is typically the value used when building an exe with Maven. + asNode().project.packaging*.value = 'pom' + } + } + } +} + +def configureTestTask(Task task, String dep, String serviceName) { + def genDir = files(tasks.named("generateTest${dep}Proto")).singleFile + def genFile = "${genDir}/grpc/io/grpc/testing/compiler/${serviceName}Grpc.java" + task.dependsOn "generateTest${dep}Proto" + task.inputs.file genFile + if (osdetector.os != 'windows') { + task.executable "diff" + task.args "-u" + } else { + task.executable "fc" + } + task.args layout.projectDirectory.file("src/test${dep}/golden/${serviceName}.java.txt") + task.args genFile + // Register an output to allow up-to-date checking + task.outputs.file(layout.buildDirectory.file(task.name)) +} + +def testGolden = tasks.register("testGolden", Exec) { + configureTestTask(it, '', 'TestService') +} +def testLiteGolden = tasks.register("testLiteGolden", Exec) { + configureTestTask(it, 'Lite', 'TestService') +} +def testDeprecatedGolden = tasks.register("testDeprecatedGolden", Exec) { + configureTestTask(it, '', 'TestDeprecatedService') +} +def testDeprecatedLiteGolden = tasks.register("testDeprecatedLiteGolden", Exec) { + configureTestTask(it, 'Lite', 'TestDeprecatedService') +} +tasks.named("test").configure { + dependsOn testGolden + dependsOn testLiteGolden + dependsOn testDeprecatedGolden + dependsOn testDeprecatedLiteGolden +} From df959ea687fe72dc7899b4c9cb6fcf711ce9cdfd Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 30 Apr 2025 16:36:28 +0000 Subject: [PATCH 12/50] missed this file in the commit --- buildscripts/make_dependencies.bat | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 5c801fa56f9..74a409de7c6 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,14 +1,18 @@ +echo on @rem set PROTOBUF_VER=21.7 +choco install -y gradle git curl pkgconfiglite +choco install -y openjdk --version=22.0.2 +set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-22.0.2\bin" +set JAVA_HOME= set PROTOBUF_VER=22.5 set ABSL_VERSION=20230125.4 -set CMAKE_NAME=cmake-3.26.3-windows-x86_64 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" ( call :installProto || exit /b 1 ) echo Compile gRPC-Java with something like: -echo -PtargetArch=x86_32 -PvcProtobufLibs=%cd%\protobuf-%PROTOBUF_VER%\build\Release -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\include +echo -PtargetArch=x86_32 -PvcProtobufLibPath=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\lib -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\include -PvcProtobufLibs=insert-list-of-libs-from-pkg-config-output-here goto :eof @@ -16,6 +20,7 @@ goto :eof where /q cmake if not ERRORLEVEL 1 goto :hasCmake +set CMAKE_NAME=cmake-3.26.3-windows-x86_64 if not exist "%CMAKE_NAME%" ( call :installCmake || exit /b 1 ) @@ -29,23 +34,26 @@ powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManag powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1 del absl.zip rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp -rename abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp +move abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp mkdir protobuf-%PROTOBUF_VER%\build pushd protobuf-%PROTOBUF_VER%\build -@rem Workaround https://github.com/protocolbuffers/protobuf/issues/10174 -powershell -command "(Get-Content ..\cmake\extract_includes.bat.in) -replace '\.\.\\', '' | Out-File -encoding ascii ..\cmake\extract_includes.bat.in" @rem cmake does not detect x86_64 from the vcvars64.bat variables. @rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS -if "%PLATFORM%" == "X64" ( +if "%PLATFORM%" == "x64" ( @rem Note the space - SET CMAKE_VSARCH= Win64 + SET CMAKE_VSARCH=-A x64 ) else ( SET CMAKE_VSARCH= ) -cmake -Dprotobuf_BUILD_TESTS=OFF -G "Visual Studio %VisualStudioVersion:~0,2%%CMAKE_VSARCH%" .. || exit /b 1 -msbuild /maxcpucount /p:Configuration=Release /verbosity:minimal libprotoc.vcxproj || exit /b 1 -call extract_includes.bat || exit /b 1 +for /f "tokens=4 delims=\" %%a in ("%VCINSTALLDIR%") do ( + SET VC_YEAR=%%a +) +for /f "tokens=1 delims=." %%a in ("%VisualStudioVersion%") do ( + SET visual_studio_major_version=%%a +) +cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%cd%\protobuf-%PROTOBUF_VER% -DCMAKE_PREFIX_PATH=%cd%\protobuf-%PROTOBUF_VER% -G "Visual Studio %visual_studio_major_version% %VC_YEAR%" %CMAKE_VSARCH% .. +cmake --build . --config Release --target install popd goto :eof @@ -56,3 +64,4 @@ powershell -command "$ErrorActionPreference = 'stop'; & { iwr https://cmake.org/ powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('cmake.zip', '.') }" || exit /b 1 del cmake.zip goto :eof + From 6ea8908698d6de933f536a0a712f88b01bf81bff Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 30 Apr 2025 16:46:19 +0000 Subject: [PATCH 13/50] Use openjdk 21 --- buildscripts/kokoro/windows32.bat | 2 +- buildscripts/kokoro/windows64.bat | 2 +- buildscripts/make_dependencies.bat | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index d14556e425a..cfa9e87168a 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -18,7 +18,7 @@ set JAVA_HOME= mkdir grpc-java-helper32 cd grpc-java-helper32 -call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars32.bat" || exit /b 1 +call "%VS140COMNTOOLS%\vsvars32.bat" || exit /b 1 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 cd "%WORKSPACE%" diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 7b91fca0a5f..26457cf7fe1 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -17,7 +17,7 @@ set JAVA_HOME= mkdir grpc-java-helper64 cd grpc-java-helper64 -call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1 +call "%VS140COMNTOOLS%\vsvars64.bat" || exit /b 1 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 cd "%WORKSPACE%" diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 74a409de7c6..bfc1005fad9 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,8 +1,8 @@ echo on @rem set PROTOBUF_VER=21.7 choco install -y gradle git curl pkgconfiglite -choco install -y openjdk --version=22.0.2 -set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-22.0.2\bin" +choco install -y openjdk --version=21.0 +set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-21\bin" set JAVA_HOME= set PROTOBUF_VER=22.5 set ABSL_VERSION=20230125.4 From 17a479b6bc288f9bfcfdc6c6a48c3482e6425d0d Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 2 May 2025 08:29:59 +0000 Subject: [PATCH 14/50] Hardcode the location of vcvars64.bat because vswhere doesn't show the VS 2022 installation path. --- buildscripts/kokoro/windows64.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 26457cf7fe1..da7a632c878 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -17,7 +17,7 @@ set JAVA_HOME= mkdir grpc-java-helper64 cd grpc-java-helper64 -call "%VS140COMNTOOLS%\vsvars64.bat" || exit /b 1 +call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 cd "%WORKSPACE%" From 4177f3eab1ce8bbe1a83234244ac4aff807476cf Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 2 May 2025 09:50:20 +0000 Subject: [PATCH 15/50] use VC 2022 install path directly in windows32.bat since vswhere doesn't show it even when it is available --- buildscripts/kokoro/windows32.bat | 2 +- buildscripts/make_dependencies.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index cfa9e87168a..d771553c619 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -18,7 +18,7 @@ set JAVA_HOME= mkdir grpc-java-helper32 cd grpc-java-helper32 -call "%VS140COMNTOOLS%\vsvars32.bat" || exit /b 1 +call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars32.bat" || exit /b 1 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 cd "%WORKSPACE%" diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index bfc1005fad9..a7e68beda55 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -2,8 +2,8 @@ echo on @rem set PROTOBUF_VER=21.7 choco install -y gradle git curl pkgconfiglite choco install -y openjdk --version=21.0 +set JAVA_HOME="c:\Program Files\OpenJDK\jdk-21" set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-21\bin" -set JAVA_HOME= set PROTOBUF_VER=22.5 set ABSL_VERSION=20230125.4 From 612ba86f0a7ecfc7ac0908cd96421414dbb3ada7 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 2 May 2025 10:33:57 +0000 Subject: [PATCH 16/50] Changing it back to using VS170COMNTOOLS --- buildscripts/kokoro/windows32.bat | 2 +- buildscripts/kokoro/windows64.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index d771553c619..d14556e425a 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -18,7 +18,7 @@ set JAVA_HOME= mkdir grpc-java-helper32 cd grpc-java-helper32 -call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars32.bat" || exit /b 1 +call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars32.bat" || exit /b 1 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 cd "%WORKSPACE%" diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index da7a632c878..1363a6ee1ea 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -17,7 +17,7 @@ set JAVA_HOME= mkdir grpc-java-helper64 cd grpc-java-helper64 -call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1 +call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 cd "%WORKSPACE%" From c19d0a53b43e6d45fc2097e7d54bc0ab72b90304 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 2 May 2025 10:45:01 +0000 Subject: [PATCH 17/50] Add powershell command = 'SilentlyContinue' because apparently protobuf progress bar is causing 'access denied error' on windows console: https://github.com/crc-org/crc/issues/1184 --- buildscripts/make_dependencies.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index a7e68beda55..91b763ef1f8 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -27,7 +27,7 @@ if not exist "%CMAKE_NAME%" ( set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin :hasCmake @rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled -powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 +powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1 del protobuf.zip powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1 From 731a8ed14003a1cda8ceb42d649be4e7027dee90 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 2 May 2025 10:52:32 +0000 Subject: [PATCH 18/50] Add powershell command = 'SilentlyContinue' because apparently protobuf progress bar is causing 'access denied error' on windows console: https://github.com/crc-org/crc/issues/1184 --- buildscripts/make_dependencies.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 91b763ef1f8..4c02562528d 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -30,7 +30,7 @@ set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1 del protobuf.zip -powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1 +powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1 del absl.zip rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp From a8da5893a42e037df890d6e277318b4c0c483d86 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 2 May 2025 12:00:15 +0000 Subject: [PATCH 19/50] Add ABSL_INTERNAL_AT_LEAST_CXX17=0 for the 64 bit archs too --- buildscripts/make_dependencies.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index e3abb0f39cc..d95984b41d5 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -62,7 +62,8 @@ else fi cmake .. \ -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 \ + -Dcrosscompile_ARCH="$GCC_ARCH" \ -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake \ -B. fi From 767c0723a6a5c76ee9b707a5ffbebc601a82022c Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 2 May 2025 14:18:07 +0000 Subject: [PATCH 20/50] Re-do lines deleted by mistake (introduced in commit 8d8e03c291f97965fec53c18400e503f0b32ed7a) to build cmake from source on 64 bit archs. --- buildscripts/make_dependencies.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index d95984b41d5..6c6e958f590 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -30,6 +30,16 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else + if [[ ! -d "cmake-${CMAKE_VERSION}" ]]; then + curl -Ls "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" | tar xz + fi + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + rm -rf "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}/bin" + cd "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}" + ./bootstrap + make + make install + ln -s /usr/local/bin/cmake /usr/bin/cmake cd "$DOWNLOAD_DIR" if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz From 17a811c18a5d77163302abb39df195e8accf49e4 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 2 May 2025 14:46:09 +0000 Subject: [PATCH 21/50] Revert last commit, fails to build cmake with OPENSSL errors, with 3.26. Needs fixing --- buildscripts/make_dependencies.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 6c6e958f590..d95984b41d5 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -30,16 +30,6 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - if [[ ! -d "cmake-${CMAKE_VERSION}" ]]; then - curl -Ls "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" | tar xz - fi - # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first - rm -rf "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}/bin" - cd "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}" - ./bootstrap - make - make install - ln -s /usr/local/bin/cmake /usr/bin/cmake cd "$DOWNLOAD_DIR" if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz From e6f445b5db47e01379fc9e6dc7052aea70e82fb4 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 5 May 2025 10:40:02 +0000 Subject: [PATCH 22/50] Use openjdk 11. --- buildscripts/kokoro/unix.sh | 2 +- buildscripts/make_dependencies.bat | 2 +- buildscripts/make_dependencies.sh | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index e65825cac01..2b2dd852419 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -34,7 +34,7 @@ cat <> gradle.properties org.gradle.jvmargs=-Xmx2048m -XX:MaxMetaspaceSize=1024m EOF -ARCH="$ARCH" buildscripts/make_dependencies.sh +#ARCH="$ARCH" buildscripts/make_dependencies.sh # Set properties via flags, do not pollute gradle.properties GRADLE_FLAGS="${GRADLE_FLAGS:-}" diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 4c02562528d..1e63352623e 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,7 +1,7 @@ echo on @rem set PROTOBUF_VER=21.7 choco install -y gradle git curl pkgconfiglite -choco install -y openjdk --version=21.0 +choco install -y openjdk --version=11.0.27 set JAVA_HOME="c:\Program Files\OpenJDK\jdk-21" set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-21\bin" set PROTOBUF_VER=22.5 diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index d95984b41d5..6c6e958f590 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -30,6 +30,16 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else + if [[ ! -d "cmake-${CMAKE_VERSION}" ]]; then + curl -Ls "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" | tar xz + fi + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + rm -rf "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}/bin" + cd "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}" + ./bootstrap + make + make install + ln -s /usr/local/bin/cmake /usr/bin/cmake cd "$DOWNLOAD_DIR" if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz From 3b97cc8fb3880f358b25847a48d42d866dd08f7b Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 5 May 2025 10:54:35 +0000 Subject: [PATCH 23/50] Remove Ctrl M characters, use gcc option to specify c++ 14 --- compiler/build.gradle | 654 +++++++++++++++++++++--------------------- 1 file changed, 327 insertions(+), 327 deletions(-) diff --git a/compiler/build.gradle b/compiler/build.gradle index 1bcfe1e5445..9883eecbbb1 100644 --- a/compiler/build.gradle +++ b/compiler/build.gradle @@ -1,327 +1,327 @@ -plugins { - id "cpp" - id "java" - id "maven-publish" - - id "com.google.protobuf" -} - -description = 'The protoc plugin for gRPC Java' - -def artifactStagingPath = "$buildDir/artifacts" as File -// Adds space-delimited arguments from the environment variable env to the -// argList. -def addEnvArgs = { env, argList -> - def value = System.getenv(env) - if (value != null) { - value.split(' +').each() { it -> argList.add(it) } - } -} - -// Adds corresponding "-l" option to the argList if libName is not found in -// LDFLAGS. This is only used for Mac because when building for uploadArchives -// artifacts, we add the ".a" files directly to LDFLAGS and without "-l" in -// order to get statically linked, otherwise we add the libraries through "-l" -// so that they can be searched for in default search paths. -def addLibraryIfNotLinked = { libName, argList -> - def ldflags = System.env.LDFLAGS - if (ldflags == null || !ldflags.contains('lib' + libName + '.a')) { - argList.add('-l' + libName) - } -} - -def String arch = rootProject.hasProperty('targetArch') ? rootProject.targetArch : osdetector.arch -def boolean vcDisable = rootProject.hasProperty('vcDisable') ? rootProject.vcDisable : false -def boolean usingVisualCpp // Whether VisualCpp is actually available and selected - -model { - toolChains { - // If you have both VC and Gcc installed, VC will be selected, unless you - // set 'vcDisable=true' - if (!vcDisable) { - visualCpp(VisualCpp) { - // Prefer vcvars-provided environment over registry-discovered environment - def String vsDir = System.getenv("VSINSTALLDIR") - def String winDir = System.getenv("WindowsSdkDir") - if (vsDir != null && winDir != null) { - installDir = vsDir - windowsSdkDir = winDir - } - } - } - gcc(Gcc) { - target("ppcle_64") { - cppCompiler.executable = 'powerpc64le-linux-gnu-g++' - linker.executable = 'powerpc64le-linux-gnu-g++' - } - target("aarch_64") { - cppCompiler.executable = 'aarch64-linux-gnu-g++' - linker.executable = 'aarch64-linux-gnu-g++' - } - target("s390_64") { - cppCompiler.executable = 's390x-linux-gnu-g++' - linker.executable = 's390x-linux-gnu-g++' - } - target("loongarch_64") - } - clang(Clang) { - target("aarch_64") {} - } - } - - platforms { - x86_32 { architecture "x86" } - x86_64 { architecture "x86_64" } - ppcle_64 { architecture "ppcle_64" } - aarch_64 { architecture "aarch_64" } - s390_64 { architecture "s390_64" } - loongarch_64 { architecture "loongarch_64" } - } - - components { - java_plugin(NativeExecutableSpec) { - if (arch in [ - 'x86_32', - 'x86_64', - 'ppcle_64', - 'aarch_64', - 's390_64', - 'loongarch_64' - ]) { - // If arch is not within the defined platforms, we do not specify the - // targetPlatform so that Gradle will choose what is appropriate. - targetPlatform arch - } - baseName "$protocPluginBaseName" - } - } - - binaries { - all { - if (toolChain in Gcc || toolChain in Clang) { - cppCompiler.define("GRPC_VERSION", version) - cppCompiler.args "--std=c++0x" - addEnvArgs("CXXFLAGS", cppCompiler.args) - addEnvArgs("CPPFLAGS", cppCompiler.args) - if (osdetector.os == "osx") { - cppCompiler.args "-mmacosx-version-min=10.7", "-stdlib=libc++" - addLibraryIfNotLinked('protoc', linker.args) - addLibraryIfNotLinked('protobuf', linker.args) - } else if (osdetector.os == "windows") { - linker.args "-static", "-lprotoc", "-lprotobuf", "-static-libgcc", "-static-libstdc++", - "-s" - } else if (osdetector.arch == "ppcle_64") { - linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-Wl,-Bdynamic", "-lpthread", "-s" - } else { - // Link protoc, protobuf, libgcc and libstdc++ statically. - // Link other (system) libraries dynamically. - // Clang under OSX doesn't support these options. - linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-static-libgcc", - "-Wl,-Bdynamic", "-lpthread", "-s" - } - addEnvArgs("LDFLAGS", linker.args) - } else if (toolChain in VisualCpp) { - usingVisualCpp = true - cppCompiler.define("GRPC_VERSION", version) - cppCompiler.args "/EHsc", "/MT" - if (rootProject.hasProperty('vcProtobufInclude')) { - cppCompiler.args "/I${rootProject.vcProtobufInclude}" - } - if (rootProject.hasProperty('vcProtobufLibs')) { - String libsList = rootProject.property('vcProtobufLibs') as String - linker.args.add("libprotoc.lib") - linker.args.add("libprotobuf.lib") - libsList.split(',').each() { lib -> linker.args.add(lib) } - } - if (rootProject.hasProperty('vcProtobufLibPaths')) { - linker.args "/LIBPATH:${rootProject.vcProtobufLibPaths}" - } - } - } - } -} - -sourceSets { - testLite { - proto { setSrcDirs(['src/test/proto']) } - } -} - -dependencies { - testImplementation project(':grpc-protobuf'), - project(':grpc-stub'), - libraries.javax.annotation - testLiteImplementation project(':grpc-protobuf-lite'), - project(':grpc-stub'), - libraries.javax.annotation -} - -tasks.named("compileTestJava").configure { - options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" -} - -tasks.named("compileTestLiteJava").configure { - options.compilerArgs = compileTestJava.options.compilerArgs - options.compilerArgs += [ - "-Xlint:-cast" - ] - options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" -} - -tasks.named("checkstyleTestLite").configure { - enabled = false -} - -protobuf { - protoc { - if (project.hasProperty('protoc')) { - path = project.protoc - } else { - artifact = libs.protobuf.protoc.get() - } - } - plugins { - grpc { path = javaPluginPath } - } - generateProtoTasks { - all().configureEach { - dependsOn 'java_pluginExecutable' - inputs.file javaPluginPath - } - ofSourceSet('test').configureEach { - plugins { grpc {} } - } - ofSourceSet('testLite').configureEach { - builtins { - java { option 'lite' } - } - plugins { - grpc { - option 'lite' - option '@generated=omit' - } - } - } - } -} - -println "*** Building codegen requires Protobuf" -println "*** Please refer to https://github.com/grpc/grpc-java/blob/master/COMPILING.md#how-to-build-code-generation-plugin" - -tasks.register("buildArtifacts", Copy) { - dependsOn 'java_pluginExecutable' - from("$buildDir/exe") { - if (osdetector.os != 'windows') { - rename 'protoc-gen-grpc-java', '$0.exe' - } - } - into artifactStagingPath -} - -base { - archivesName = "$protocPluginBaseName" -} - -def checkArtifacts = tasks.register("checkArtifacts") { - dependsOn buildArtifacts - doLast { - if (!usingVisualCpp) { - def ret = exec { - executable 'bash' - args 'check-artifact.sh', osdetector.os, arch - } - if (ret.exitValue != 0) { - throw new GradleException("check-artifact.sh exited with " + ret.exitValue) - } - } else { - def exeName = "$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" - def os = new ByteArrayOutputStream() - def ret = exec { - executable 'dumpbin' - args '/nologo', '/dependents', exeName - standardOutput = os - } - if (ret.exitValue != 0) { - throw new GradleException("dumpbin exited with " + ret.exitValue) - } - def dlls = os.toString() =~ /Image has the following dependencies:\s+(.*)\s+Summary/ - if (dlls[0][1] != "KERNEL32.dll") { - throw new Exception("unexpected dll deps: " + dlls[0][1]); - } - os.reset() - ret = exec { - executable 'dumpbin' - args '/nologo', '/headers', exeName - standardOutput = os - } - if (ret.exitValue != 0) { - throw new GradleException("dumpbin exited with " + ret.exitValue) - } - def machine = os.toString() =~ / machine \(([^)]+)\)/ - def expectedArch = [x86_32: "x86", x86_64: "x64"][arch] - if (machine[0][1] != expectedArch) { - throw new Exception("unexpected architecture: " + machine[0][1]); - } - } - } -} - -// Exe files are skipped by Maven by default. Override it. -// Also skip jar files that is generated by the java plugin. -publishing { - publications { - maven(MavenPublication) { - // Removes all artifacts since grpc-compiler doesn't generates any Jar - artifacts = [] - artifactId 'protoc-gen-grpc-java' - artifact("$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" as File) { - classifier osdetector.os + "-" + arch - extension "exe" - builtBy checkArtifacts - } - pom.withXml { - // This isn't any sort of Java archive artifact, and OSSRH doesn't enforce - // javadoc for 'pom' packages. 'exe' would be a more appropriate packaging - // value, but it isn't clear how that will be interpreted. In addition, - // 'pom' is typically the value used when building an exe with Maven. - asNode().project.packaging*.value = 'pom' - } - } - } -} - -def configureTestTask(Task task, String dep, String serviceName) { - def genDir = files(tasks.named("generateTest${dep}Proto")).singleFile - def genFile = "${genDir}/grpc/io/grpc/testing/compiler/${serviceName}Grpc.java" - task.dependsOn "generateTest${dep}Proto" - task.inputs.file genFile - if (osdetector.os != 'windows') { - task.executable "diff" - task.args "-u" - } else { - task.executable "fc" - } - task.args layout.projectDirectory.file("src/test${dep}/golden/${serviceName}.java.txt") - task.args genFile - // Register an output to allow up-to-date checking - task.outputs.file(layout.buildDirectory.file(task.name)) -} - -def testGolden = tasks.register("testGolden", Exec) { - configureTestTask(it, '', 'TestService') -} -def testLiteGolden = tasks.register("testLiteGolden", Exec) { - configureTestTask(it, 'Lite', 'TestService') -} -def testDeprecatedGolden = tasks.register("testDeprecatedGolden", Exec) { - configureTestTask(it, '', 'TestDeprecatedService') -} -def testDeprecatedLiteGolden = tasks.register("testDeprecatedLiteGolden", Exec) { - configureTestTask(it, 'Lite', 'TestDeprecatedService') -} -tasks.named("test").configure { - dependsOn testGolden - dependsOn testLiteGolden - dependsOn testDeprecatedGolden - dependsOn testDeprecatedLiteGolden -} +plugins { + id "cpp" + id "java" + id "maven-publish" + + id "com.google.protobuf" +} + +description = 'The protoc plugin for gRPC Java' + +def artifactStagingPath = "$buildDir/artifacts" as File +// Adds space-delimited arguments from the environment variable env to the +// argList. +def addEnvArgs = { env, argList -> + def value = System.getenv(env) + if (value != null) { + value.split(' +').each() { it -> argList.add(it) } + } +} + +// Adds corresponding "-l" option to the argList if libName is not found in +// LDFLAGS. This is only used for Mac because when building for uploadArchives +// artifacts, we add the ".a" files directly to LDFLAGS and without "-l" in +// order to get statically linked, otherwise we add the libraries through "-l" +// so that they can be searched for in default search paths. +def addLibraryIfNotLinked = { libName, argList -> + def ldflags = System.env.LDFLAGS + if (ldflags == null || !ldflags.contains('lib' + libName + '.a')) { + argList.add('-l' + libName) + } +} + +def String arch = rootProject.hasProperty('targetArch') ? rootProject.targetArch : osdetector.arch +def boolean vcDisable = rootProject.hasProperty('vcDisable') ? rootProject.vcDisable : false +def boolean usingVisualCpp // Whether VisualCpp is actually available and selected + +model { + toolChains { + // If you have both VC and Gcc installed, VC will be selected, unless you + // set 'vcDisable=true' + if (!vcDisable) { + visualCpp(VisualCpp) { + // Prefer vcvars-provided environment over registry-discovered environment + def String vsDir = System.getenv("VSINSTALLDIR") + def String winDir = System.getenv("WindowsSdkDir") + if (vsDir != null && winDir != null) { + installDir = vsDir + windowsSdkDir = winDir + } + } + } + gcc(Gcc) { + target("ppcle_64") { + cppCompiler.executable = 'powerpc64le-linux-gnu-g++' + linker.executable = 'powerpc64le-linux-gnu-g++' + } + target("aarch_64") { + cppCompiler.executable = 'aarch64-linux-gnu-g++' + linker.executable = 'aarch64-linux-gnu-g++' + } + target("s390_64") { + cppCompiler.executable = 's390x-linux-gnu-g++' + linker.executable = 's390x-linux-gnu-g++' + } + target("loongarch_64") + } + clang(Clang) { + target("aarch_64") {} + } + } + + platforms { + x86_32 { architecture "x86" } + x86_64 { architecture "x86_64" } + ppcle_64 { architecture "ppcle_64" } + aarch_64 { architecture "aarch_64" } + s390_64 { architecture "s390_64" } + loongarch_64 { architecture "loongarch_64" } + } + + components { + java_plugin(NativeExecutableSpec) { + if (arch in [ + 'x86_32', + 'x86_64', + 'ppcle_64', + 'aarch_64', + 's390_64', + 'loongarch_64' + ]) { + // If arch is not within the defined platforms, we do not specify the + // targetPlatform so that Gradle will choose what is appropriate. + targetPlatform arch + } + baseName "$protocPluginBaseName" + } + } + + binaries { + all { + if (toolChain in Gcc || toolChain in Clang) { + cppCompiler.define("GRPC_VERSION", version) + cppCompiler.args "--std=c++14" + addEnvArgs("CXXFLAGS", cppCompiler.args) + addEnvArgs("CPPFLAGS", cppCompiler.args) + if (osdetector.os == "osx") { + cppCompiler.args "-mmacosx-version-min=10.7", "-stdlib=libc++" + addLibraryIfNotLinked('protoc', linker.args) + addLibraryIfNotLinked('protobuf', linker.args) + } else if (osdetector.os == "windows") { + linker.args "-static", "-lprotoc", "-lprotobuf", "-static-libgcc", "-static-libstdc++", + "-s" + } else if (osdetector.arch == "ppcle_64") { + linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-Wl,-Bdynamic", "-lpthread", "-s" + } else { + // Link protoc, protobuf, libgcc and libstdc++ statically. + // Link other (system) libraries dynamically. + // Clang under OSX doesn't support these options. + linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-static-libgcc", + "-Wl,-Bdynamic", "-lpthread", "-s" + } + addEnvArgs("LDFLAGS", linker.args) + } else if (toolChain in VisualCpp) { + usingVisualCpp = true + cppCompiler.define("GRPC_VERSION", version) + cppCompiler.args "/EHsc", "/MT" + if (rootProject.hasProperty('vcProtobufInclude')) { + cppCompiler.args "/I${rootProject.vcProtobufInclude}" + } + if (rootProject.hasProperty('vcProtobufLibs')) { + String libsList = rootProject.property('vcProtobufLibs') as String + linker.args.add("libprotoc.lib") + linker.args.add("libprotobuf.lib") + libsList.split(',').each() { lib -> linker.args.add(lib) } + } + if (rootProject.hasProperty('vcProtobufLibPaths')) { + linker.args "/LIBPATH:${rootProject.vcProtobufLibPaths}" + } + } + } + } +} + +sourceSets { + testLite { + proto { setSrcDirs(['src/test/proto']) } + } +} + +dependencies { + testImplementation project(':grpc-protobuf'), + project(':grpc-stub'), + libraries.javax.annotation + testLiteImplementation project(':grpc-protobuf-lite'), + project(':grpc-stub'), + libraries.javax.annotation +} + +tasks.named("compileTestJava").configure { + options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" +} + +tasks.named("compileTestLiteJava").configure { + options.compilerArgs = compileTestJava.options.compilerArgs + options.compilerArgs += [ + "-Xlint:-cast" + ] + options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" +} + +tasks.named("checkstyleTestLite").configure { + enabled = false +} + +protobuf { + protoc { + if (project.hasProperty('protoc')) { + path = project.protoc + } else { + artifact = libs.protobuf.protoc.get() + } + } + plugins { + grpc { path = javaPluginPath } + } + generateProtoTasks { + all().configureEach { + dependsOn 'java_pluginExecutable' + inputs.file javaPluginPath + } + ofSourceSet('test').configureEach { + plugins { grpc {} } + } + ofSourceSet('testLite').configureEach { + builtins { + java { option 'lite' } + } + plugins { + grpc { + option 'lite' + option '@generated=omit' + } + } + } + } +} + +println "*** Building codegen requires Protobuf" +println "*** Please refer to https://github.com/grpc/grpc-java/blob/master/COMPILING.md#how-to-build-code-generation-plugin" + +tasks.register("buildArtifacts", Copy) { + dependsOn 'java_pluginExecutable' + from("$buildDir/exe") { + if (osdetector.os != 'windows') { + rename 'protoc-gen-grpc-java', '$0.exe' + } + } + into artifactStagingPath +} + +base { + archivesName = "$protocPluginBaseName" +} + +def checkArtifacts = tasks.register("checkArtifacts") { + dependsOn buildArtifacts + doLast { + if (!usingVisualCpp) { + def ret = exec { + executable 'bash' + args 'check-artifact.sh', osdetector.os, arch + } + if (ret.exitValue != 0) { + throw new GradleException("check-artifact.sh exited with " + ret.exitValue) + } + } else { + def exeName = "$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" + def os = new ByteArrayOutputStream() + def ret = exec { + executable 'dumpbin' + args '/nologo', '/dependents', exeName + standardOutput = os + } + if (ret.exitValue != 0) { + throw new GradleException("dumpbin exited with " + ret.exitValue) + } + def dlls = os.toString() =~ /Image has the following dependencies:\s+(.*)\s+Summary/ + if (dlls[0][1] != "KERNEL32.dll") { + throw new Exception("unexpected dll deps: " + dlls[0][1]); + } + os.reset() + ret = exec { + executable 'dumpbin' + args '/nologo', '/headers', exeName + standardOutput = os + } + if (ret.exitValue != 0) { + throw new GradleException("dumpbin exited with " + ret.exitValue) + } + def machine = os.toString() =~ / machine \(([^)]+)\)/ + def expectedArch = [x86_32: "x86", x86_64: "x64"][arch] + if (machine[0][1] != expectedArch) { + throw new Exception("unexpected architecture: " + machine[0][1]); + } + } + } +} + +// Exe files are skipped by Maven by default. Override it. +// Also skip jar files that is generated by the java plugin. +publishing { + publications { + maven(MavenPublication) { + // Removes all artifacts since grpc-compiler doesn't generates any Jar + artifacts = [] + artifactId 'protoc-gen-grpc-java' + artifact("$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" as File) { + classifier osdetector.os + "-" + arch + extension "exe" + builtBy checkArtifacts + } + pom.withXml { + // This isn't any sort of Java archive artifact, and OSSRH doesn't enforce + // javadoc for 'pom' packages. 'exe' would be a more appropriate packaging + // value, but it isn't clear how that will be interpreted. In addition, + // 'pom' is typically the value used when building an exe with Maven. + asNode().project.packaging*.value = 'pom' + } + } + } +} + +def configureTestTask(Task task, String dep, String serviceName) { + def genDir = files(tasks.named("generateTest${dep}Proto")).singleFile + def genFile = "${genDir}/grpc/io/grpc/testing/compiler/${serviceName}Grpc.java" + task.dependsOn "generateTest${dep}Proto" + task.inputs.file genFile + if (osdetector.os != 'windows') { + task.executable "diff" + task.args "-u" + } else { + task.executable "fc" + } + task.args layout.projectDirectory.file("src/test${dep}/golden/${serviceName}.java.txt") + task.args genFile + // Register an output to allow up-to-date checking + task.outputs.file(layout.buildDirectory.file(task.name)) +} + +def testGolden = tasks.register("testGolden", Exec) { + configureTestTask(it, '', 'TestService') +} +def testLiteGolden = tasks.register("testLiteGolden", Exec) { + configureTestTask(it, 'Lite', 'TestService') +} +def testDeprecatedGolden = tasks.register("testDeprecatedGolden", Exec) { + configureTestTask(it, '', 'TestDeprecatedService') +} +def testDeprecatedLiteGolden = tasks.register("testDeprecatedLiteGolden", Exec) { + configureTestTask(it, 'Lite', 'TestDeprecatedService') +} +tasks.named("test").configure { + dependsOn testGolden + dependsOn testLiteGolden + dependsOn testDeprecatedGolden + dependsOn testDeprecatedLiteGolden +} From a2fd8dc2f8bba98c5e77ed16149c3c1683ec84be Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 5 May 2025 11:05:52 +0000 Subject: [PATCH 24/50] Fix open jdk version to 11.0 as 11.0.27 is not available. --- buildscripts/make_dependencies.bat | 2 +- buildscripts/make_dependencies.sh | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 1e63352623e..10b587be13c 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,7 +1,7 @@ echo on @rem set PROTOBUF_VER=21.7 choco install -y gradle git curl pkgconfiglite -choco install -y openjdk --version=11.0.27 +choco install -y openjdk --version=11.0 set JAVA_HOME="c:\Program Files\OpenJDK\jdk-21" set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-21\bin" set PROTOBUF_VER=22.5 diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 6c6e958f590..b330e033e16 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -30,17 +30,6 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - if [[ ! -d "cmake-${CMAKE_VERSION}" ]]; then - curl -Ls "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" | tar xz - fi - # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first - rm -rf "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}/bin" - cd "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}" - ./bootstrap - make - make install - ln -s /usr/local/bin/cmake /usr/bin/cmake - cd "$DOWNLOAD_DIR" if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz From 5b96e1e007ba3573080104b5c7049918d34dee1c Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 5 May 2025 11:26:51 +0000 Subject: [PATCH 25/50] Fix jdk version --- buildscripts/make_dependencies.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 10b587be13c..dc073508540 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,9 +1,9 @@ echo on @rem set PROTOBUF_VER=21.7 choco install -y gradle git curl pkgconfiglite -choco install -y openjdk --version=11.0 -set JAVA_HOME="c:\Program Files\OpenJDK\jdk-21" -set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-21\bin" +choco install -y openjdk --version=17.0 +set JAVA_HOME="c:\Program Files\OpenJDK\jdk-17" +set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-17\bin" set PROTOBUF_VER=22.5 set ABSL_VERSION=20230125.4 From 8c35eeded7da970e68d038dbc3568a4f969989f2 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 5 May 2025 11:48:40 +0000 Subject: [PATCH 26/50] Remove accidental commenting out. --- buildscripts/kokoro/unix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index 2b2dd852419..e65825cac01 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -34,7 +34,7 @@ cat <> gradle.properties org.gradle.jvmargs=-Xmx2048m -XX:MaxMetaspaceSize=1024m EOF -#ARCH="$ARCH" buildscripts/make_dependencies.sh +ARCH="$ARCH" buildscripts/make_dependencies.sh # Set properties via flags, do not pollute gradle.properties GRADLE_FLAGS="${GRADLE_FLAGS:-}" From f91dbc70bbc7c14b06f78788de7914e7aced33d4 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 5 May 2025 12:02:33 +0000 Subject: [PATCH 27/50] Missed libs --- buildscripts/kokoro/windows32.bat | 2 +- buildscripts/kokoro/windows64.bat | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index d14556e425a..3eb2277a78c 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -30,7 +30,7 @@ SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\ SET VC_PROTOBUF_LIB_PATHS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include call :Get_Libs -SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true +SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% build" diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 1363a6ee1ea..760e942c1b8 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -17,7 +17,7 @@ set JAVA_HOME= mkdir grpc-java-helper64 cd grpc-java-helper64 -call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1 +call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 cd "%WORKSPACE%" @@ -29,7 +29,7 @@ SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\ SET VC_PROTOBUF_LIB_PATHS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include call :Get_Libs -SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true +SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" @rem make sure no daemons have any files open From 19d0917734f873a1b21333d353f0ceba6b65d398 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 5 May 2025 12:26:08 +0000 Subject: [PATCH 28/50] Remove leading whitespace in libraries list --- buildscripts/kokoro/windows32.bat | 6 +++++- buildscripts/kokoro/windows64.bat | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index 3eb2277a78c..0b8de381eda 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -69,7 +69,11 @@ for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( @rem Because protobuf is specified as libprotobuf and elsewhere if !lib! NEQ protobuf ( set lib=!lib!.lib, - set libs_list=!libs_list! !lib! + if "%libs_list%"=="" ( + set libs_list=!lib! + ) else ( + set libs_list=!libs_list! !lib! + ) ) ) ) diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 760e942c1b8..4d56fcf224e 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -53,7 +53,11 @@ for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( @rem Because protobuf is specified as libprotobuf and elsewhere if !lib! NEQ protobuf ( set lib=!lib!.lib, - set libs_list=!libs_list! !lib! + if "%libs_list%"=="" ( + set libs_list=!lib! + ) else ( + set libs_list=!libs_list! !lib! + ) ) ) ) From 4d3f2522f812692b74ee2763aa0b520b7cd82a05 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 5 May 2025 13:40:15 +0000 Subject: [PATCH 29/50] Fix libs leading space --- buildscripts/kokoro/windows64.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 4d56fcf224e..c0934d53595 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -53,7 +53,7 @@ for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( @rem Because protobuf is specified as libprotobuf and elsewhere if !lib! NEQ protobuf ( set lib=!lib!.lib, - if "%libs_list%"=="" ( + if "!libs_list!"=="" ( set libs_list=!lib! ) else ( set libs_list=!libs_list! !lib! From 02788d8c14e454b5635702dfda32275ddf2e7c2d Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 07:02:41 +0000 Subject: [PATCH 30/50] Don't put spaces in between libs since it breaks the env variable value passed to the command. --- buildscripts/kokoro/windows32.bat | 4 ++-- buildscripts/kokoro/windows64.bat | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index 0b8de381eda..ee80e67bfaf 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -68,11 +68,11 @@ for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( set lib=!lib: =! @rem Because protobuf is specified as libprotobuf and elsewhere if !lib! NEQ protobuf ( - set lib=!lib!.lib, + set lib=!lib!.lib if "%libs_list%"=="" ( set libs_list=!lib! ) else ( - set libs_list=!libs_list! !lib! + set libs_list=!libs_list!,!lib! ) ) ) diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index c0934d53595..374da20eb4f 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -52,11 +52,11 @@ for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( set lib=!lib: =! @rem Because protobuf is specified as libprotobuf and elsewhere if !lib! NEQ protobuf ( - set lib=!lib!.lib, + set lib=!lib!.lib if "!libs_list!"=="" ( set libs_list=!lib! ) else ( - set libs_list=!libs_list! !lib! + set libs_list=!libs_list!,!lib! ) ) ) From f6fa6f0ac5682041ef36505ee3183e2269ce98d2 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 07:14:07 +0000 Subject: [PATCH 31/50] Download cmake for android --- buildscripts/kokoro/android.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 08e564458a3..5bb6831e894 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -30,7 +30,10 @@ unzip -qd "${ANDROID_HOME}/cmdline-tools" cmdline.zip rm cmdline.zip mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/latest" (yes || true) | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --licenses - +curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local +export PATH=/var/local/cmake-3.26.3-linux-x86_64/bin:$PATH + # Proto deps buildscripts/make_dependencies.sh From c3bdce75792ae3425c2a17db780cfff68f988872 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 07:21:12 +0000 Subject: [PATCH 32/50] Change from /var/local to /usr/local to download cmake as the path is not writable. --- buildscripts/kokoro/android.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 5bb6831e894..618cef3439e 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -31,8 +31,8 @@ rm cmdline.zip mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/latest" (yes || true) | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --licenses curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ - tar xz -C /var/local -export PATH=/var/local/cmake-3.26.3-linux-x86_64/bin:$PATH + tar xz -C /usr/local +export PATH=/usr/local/cmake-3.26.3-linux-x86_64/bin:$PATH # Proto deps buildscripts/make_dependencies.sh From 927fca6ad25abbe3e1d9d58bd425b5cfa13e6ccc Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 07:31:41 +0000 Subject: [PATCH 33/50] Fix forgotten % instead of ! for local scope variable. --- buildscripts/kokoro/windows32.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index ee80e67bfaf..2cce825237d 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -69,7 +69,7 @@ for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( @rem Because protobuf is specified as libprotobuf and elsewhere if !lib! NEQ protobuf ( set lib=!lib!.lib - if "%libs_list%"=="" ( + if "!libs_list!"=="" ( set libs_list=!lib! ) else ( set libs_list=!libs_list!,!lib! From 8d5afa13373de9fd97c0833c81e9f7894b11cd13 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 07:37:26 +0000 Subject: [PATCH 34/50] Can't write to /usr/local either, use /tmp to install cmake for android. --- buildscripts/kokoro/android.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 618cef3439e..e28d1c98571 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -31,8 +31,8 @@ rm cmdline.zip mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/latest" (yes || true) | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --licenses curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ - tar xz -C /usr/local -export PATH=/usr/local/cmake-3.26.3-linux-x86_64/bin:$PATH + tar xz -C /tmp +export PATH=/tmp/cmake-3.26.3-linux-x86_64/bin:$PATH # Proto deps buildscripts/make_dependencies.sh From bc57b031aa74367021235614c1a0cf874625f300 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 07:49:24 +0000 Subject: [PATCH 35/50] Install pkg-config for android --- buildscripts/kokoro/android.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index e28d1c98571..a3882452293 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -33,6 +33,7 @@ mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/ curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ tar xz -C /tmp export PATH=/tmp/cmake-3.26.3-linux-x86_64/bin:$PATH +sudo apt-get install pkg-config # Proto deps buildscripts/make_dependencies.sh From 75c2a7dd7e6de27767619ed22f1438cf3daa5b91 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 09:36:19 +0000 Subject: [PATCH 36/50] Fix x32 VS generator --- compiler/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/build.gradle b/compiler/build.gradle index 9883eecbbb1..077dc9ebf2b 100644 --- a/compiler/build.gradle +++ b/compiler/build.gradle @@ -127,12 +127,12 @@ model { if (rootProject.hasProperty('vcProtobufInclude')) { cppCompiler.args "/I${rootProject.vcProtobufInclude}" } + linker.args.add("libprotoc.lib") + linker.args.add("libprotobuf.lib") if (rootProject.hasProperty('vcProtobufLibs')) { String libsList = rootProject.property('vcProtobufLibs') as String - linker.args.add("libprotoc.lib") - linker.args.add("libprotobuf.lib") libsList.split(',').each() { lib -> linker.args.add(lib) } - } + } if (rootProject.hasProperty('vcProtobufLibPaths')) { linker.args "/LIBPATH:${rootProject.vcProtobufLibPaths}" } From 1369473df6fff7f64d07cc003aed0a022445cca1 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 09:43:16 +0000 Subject: [PATCH 37/50] apt-get update before install pkg-config. --- buildscripts/kokoro/android.sh | 2 +- buildscripts/make_dependencies.bat | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index a3882452293..e1454b94e30 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -33,7 +33,7 @@ mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/ curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ tar xz -C /tmp export PATH=/tmp/cmake-3.26.3-linux-x86_64/bin:$PATH -sudo apt-get install pkg-config +sudo apt-get update && sudo apt-get install pkg-config # Proto deps buildscripts/make_dependencies.sh diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index dc073508540..660ca635cfd 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -39,10 +39,11 @@ mkdir protobuf-%PROTOBUF_VER%\build pushd protobuf-%PROTOBUF_VER%\build @rem cmake does not detect x86_64 from the vcvars64.bat variables. -@rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS +@rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS, likewise for x32 if "%PLATFORM%" == "x64" ( - @rem Note the space SET CMAKE_VSARCH=-A x64 +) else if "%PLATFORM%" == "x32" ( + SET CMAKE_VSARCH=-A x32 ) else ( SET CMAKE_VSARCH= ) @@ -53,7 +54,7 @@ for /f "tokens=1 delims=." %%a in ("%VisualStudioVersion%") do ( SET visual_studio_major_version=%%a ) cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%cd%\protobuf-%PROTOBUF_VER% -DCMAKE_PREFIX_PATH=%cd%\protobuf-%PROTOBUF_VER% -G "Visual Studio %visual_studio_major_version% %VC_YEAR%" %CMAKE_VSARCH% .. -cmake --build . --config Release --target install +cmake --build . --config Release --target install popd goto :eof From 260fccf05e684669dba8db2307009b703400b008 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 14:14:55 +0000 Subject: [PATCH 38/50] Fix mistake in order of statements reg. pkg-config --- buildscripts/kokoro/android.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index e1454b94e30..682a0195802 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -9,9 +9,6 @@ BASE_DIR="$(pwd)" cd "$BASE_DIR/github/grpc-java" -export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" -export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" -export LD_LIBRARY_PATH=/tmp/protobuf/lib export OS_NAME=$(uname) cat <> gradle.properties @@ -33,11 +30,15 @@ mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/ curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ tar xz -C /tmp export PATH=/tmp/cmake-3.26.3-linux-x86_64/bin:$PATH -sudo apt-get update && sudo apt-get install pkg-config # Proto deps buildscripts/make_dependencies.sh +sudo apt-get update && sudo apt-get install pkg-config +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" +export LD_LIBRARY_PATH=/tmp/protobuf/lib + # Build Android with Java 11, this adds it to the PATH sudo update-java-alternatives --set java-1.11.0-openjdk-amd64 # Unset any existing JAVA_HOME env var to stop Gradle from using it From 4ef7fd1d938572b5426a4c74dbb245f3226f2147 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 14:21:29 +0000 Subject: [PATCH 39/50] Fix mistaken use of x32 instead of x86 --- buildscripts/make_dependencies.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 660ca635cfd..fcb73d52886 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -42,7 +42,7 @@ pushd protobuf-%PROTOBUF_VER%\build @rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS, likewise for x32 if "%PLATFORM%" == "x64" ( SET CMAKE_VSARCH=-A x64 -) else if "%PLATFORM%" == "x32" ( +) else if "%PLATFORM%" == "x86" ( SET CMAKE_VSARCH=-A x32 ) else ( SET CMAKE_VSARCH= From 088b489a6cb54c3dab85f9c54f31c0d373b95b35 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Tue, 6 May 2025 15:48:21 +0000 Subject: [PATCH 40/50] Fix typo --- buildscripts/make_dependencies.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index fcb73d52886..d7a8306cdcf 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -43,7 +43,7 @@ pushd protobuf-%PROTOBUF_VER%\build if "%PLATFORM%" == "x64" ( SET CMAKE_VSARCH=-A x64 ) else if "%PLATFORM%" == "x86" ( - SET CMAKE_VSARCH=-A x32 + SET CMAKE_VSARCH=-A x86 ) else ( SET CMAKE_VSARCH= ) From f4aeb67464f2bef5d072fd0475d7bbdf4bb79d99 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 7 May 2025 05:02:48 +0000 Subject: [PATCH 41/50] Skip codegen when rebuilding HEAD. --- buildscripts/kokoro/android.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 682a0195802..8a5eca33b06 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -103,6 +103,7 @@ cd $BASE_DIR/github/grpc-java ./gradlew clean git checkout HEAD^ ./gradlew --stop # use a new daemon to build the previous commit +GRADLE_FLAGS = $GRADLE_FLAGS + " -PskipCodegen=true" ./gradlew publishToMavenLocal $GRADLE_FLAGS cd examples/android/helloworld/ ../../gradlew build $GRADLE_FLAGS From 09535d86e0c78fd868f874f27d14d875ccb43f78 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 7 May 2025 05:49:35 +0000 Subject: [PATCH 42/50] Remove syntax error in assignment --- buildscripts/kokoro/android.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 8a5eca33b06..3d9cbf0769b 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -103,7 +103,7 @@ cd $BASE_DIR/github/grpc-java ./gradlew clean git checkout HEAD^ ./gradlew --stop # use a new daemon to build the previous commit -GRADLE_FLAGS = $GRADLE_FLAGS + " -PskipCodegen=true" +GRADLE_FLAGS=$GRADLE_FLAGS + " -PskipCodegen=true" ./gradlew publishToMavenLocal $GRADLE_FLAGS cd examples/android/helloworld/ ../../gradlew build $GRADLE_FLAGS From 243578078b4334d62ddaa78e2ac3665119c0173a Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 7 May 2025 06:09:27 +0000 Subject: [PATCH 43/50] Remove syntax error in assignment --- buildscripts/kokoro/android.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 3d9cbf0769b..8452f451d69 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -103,7 +103,7 @@ cd $BASE_DIR/github/grpc-java ./gradlew clean git checkout HEAD^ ./gradlew --stop # use a new daemon to build the previous commit -GRADLE_FLAGS=$GRADLE_FLAGS + " -PskipCodegen=true" +GRADLE_FLAGS="${GRADLE_FLAGS} -PskipCodegen=true" ./gradlew publishToMavenLocal $GRADLE_FLAGS cd examples/android/helloworld/ ../../gradlew build $GRADLE_FLAGS From 88999f850617de71749a00ad0098d48d048c2c2f Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 7 May 2025 07:22:18 +0000 Subject: [PATCH 44/50] Windows geneator flag for x86 --- buildscripts/make_dependencies.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index d7a8306cdcf..96dce363f23 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -43,7 +43,8 @@ pushd protobuf-%PROTOBUF_VER%\build if "%PLATFORM%" == "x64" ( SET CMAKE_VSARCH=-A x64 ) else if "%PLATFORM%" == "x86" ( - SET CMAKE_VSARCH=-A x86 + @rem -A x86 doesn't work: https://github.com/microsoft/vcpkg/issues/15465 + SET CMAKE_VSARCH=-DCMAKE_GENERATOR_PLATFORM=WIN32 ) else ( SET CMAKE_VSARCH= ) From 8e16a15073e583eaa0770e71b4d42d6053b02c6b Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 7 May 2025 11:49:52 +0000 Subject: [PATCH 45/50] Install pkg-config on mac --- buildscripts/kokoro/macos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/kokoro/macos.sh b/buildscripts/kokoro/macos.sh index 018d15dd2f9..7aad72c838c 100755 --- a/buildscripts/kokoro/macos.sh +++ b/buildscripts/kokoro/macos.sh @@ -17,5 +17,6 @@ trap spongify_logs EXIT export -n JAVA_HOME export PATH="$(/usr/libexec/java_home -v"1.8.0")/bin:${PATH}" +sudo apt-get update && sudo apt-get install pkg-config "$GRPC_JAVA_DIR"/buildscripts/kokoro/unix.sh From 4f7c9bb20b60ee5dc74ed72f572b40946e0a8205 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 7 May 2025 13:59:11 +0000 Subject: [PATCH 46/50] Change apt-get to brew for mac os for installing pkg-config --- buildscripts/kokoro/macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/kokoro/macos.sh b/buildscripts/kokoro/macos.sh index 7aad72c838c..b7885362daa 100755 --- a/buildscripts/kokoro/macos.sh +++ b/buildscripts/kokoro/macos.sh @@ -17,6 +17,6 @@ trap spongify_logs EXIT export -n JAVA_HOME export PATH="$(/usr/libexec/java_home -v"1.8.0")/bin:${PATH}" -sudo apt-get update && sudo apt-get install pkg-config +brew install pkg-config "$GRPC_JAVA_DIR"/buildscripts/kokoro/unix.sh From 3c8bc900c71715cf0234202df610df9d4db15bae Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 7 May 2025 17:11:35 +0000 Subject: [PATCH 47/50] Remove installing pkg-config, its alreay there. --- buildscripts/kokoro/macos.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/buildscripts/kokoro/macos.sh b/buildscripts/kokoro/macos.sh index b7885362daa..018d15dd2f9 100755 --- a/buildscripts/kokoro/macos.sh +++ b/buildscripts/kokoro/macos.sh @@ -17,6 +17,5 @@ trap spongify_logs EXIT export -n JAVA_HOME export PATH="$(/usr/libexec/java_home -v"1.8.0")/bin:${PATH}" -brew install pkg-config "$GRPC_JAVA_DIR"/buildscripts/kokoro/unix.sh From 415e913dea5145d6688a7475428e1860625a2ac3 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 7 May 2025 17:28:02 +0000 Subject: [PATCH 48/50] Try LD_LIBRARY_PATH as well, for mac since I'm getting linker errors with LIBRARY_PATH still. --- buildscripts/make_dependencies.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index b330e033e16..4967b531d92 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -86,4 +86,5 @@ To compile with the build dependencies: export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LIBRARY_PATH=/tmp/protobuf/lib +export LD_LIBRARY_PATH=/tmp/protobuf/lib EOF From e529cf1320ef7c7807f6ee7ec17e99d30eaca494 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 9 May 2025 12:41:52 +0000 Subject: [PATCH 49/50] Skip failing socket options test for windows --- okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java b/okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java index 895ba7ff7c7..ede8511ee70 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java @@ -25,6 +25,7 @@ import io.grpc.okhttp.internal.TlsVersion; import java.net.Socket; import java.util.List; +import java.util.Locale; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -95,6 +96,9 @@ public void getSocketOptions() throws Exception { assertEquals("5000", socketOptions.others.get("SO_SNDBUF")); assertEquals("true", socketOptions.others.get("SO_KEEPALIVE")); assertEquals("true", socketOptions.others.get("SO_OOBINLINE")); - assertEquals("8", socketOptions.others.get("IP_TOS")); + String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); + if (!osName.startsWith("windows")) { + assertEquals("8", socketOptions.others.get("IP_TOS")); + } } } From 5280d65a795decbee461ca222e93cd55ee30603a Mon Sep 17 00:00:00 2001 From: Kannan J Date: Fri, 9 May 2025 14:46:00 +0000 Subject: [PATCH 50/50] Add patch for absl time CMakeLists --- buildscripts/absl_time_CMakeLists.patch | 9 +++++++++ buildscripts/make_dependencies.sh | 1 + 2 files changed, 10 insertions(+) create mode 100644 buildscripts/absl_time_CMakeLists.patch diff --git a/buildscripts/absl_time_CMakeLists.patch b/buildscripts/absl_time_CMakeLists.patch new file mode 100644 index 00000000000..4260a967fdd --- /dev/null +++ b/buildscripts/absl_time_CMakeLists.patch @@ -0,0 +1,9 @@ +57,60d56 +< if(APPLE) +< find_library(CoreFoundation CoreFoundation) +< endif() +< +87c83 +< $<$:${CoreFoundation}> +--- +> $<$:-Wl,-framework,CoreFoundation> diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 4967b531d92..ccd7e05cf5a 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -34,6 +34,7 @@ else curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz rmdir "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" + patch abseil-cpp-$ABSL_VERSION/absl/time/CMakeLists.txt $BUILDSCRIPTS_DIR/absl_time_CMakeLists.patch mv "abseil-cpp-$ABSL_VERSION" "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" fi # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first