Skip to content

Commit 46485c8

Browse files
kannanjgithubkannanjgithub
authored
Upgrade Protobuf C++ to 22.5 (#11961)
1 parent 9406d3b commit 46485c8

16 files changed

+206
-66
lines changed

‎COMPILING.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code
4444
generation. Most users only need to use `skipCodegen=true` as discussed above.
4545

4646
### Build Protobuf
47-
The codegen plugin is C++ code and requires protobuf 21.7 or later.
47+
The codegen plugin is C++ code and requires protobuf 22.5 or later.
4848

4949
For Linux, Mac and MinGW:
5050
```
51-
$ PROTOBUF_VERSION=21.7
51+
$ PROTOBUF_VERSION=22.5
5252
$ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz
5353
$ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz
5454
$ cd protobuf-$PROTOBUF_VERSION

‎buildscripts/grpc-java-artifacts/Dockerfile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,11 @@ RUN mkdir -p "$ANDROID_HOME/cmdline-tools" && \
2727
mv "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest" && \
2828
yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses
2929

30+
RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
31+
tar xz -C /var/local
32+
3033
# Install Maven
3134
RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \
3235
tar xz -C /var/local
33-
ENV PATH /var/local/apache-maven-3.8.8/bin:$PATH
36+
ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH
37+

‎buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,11 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
1010
g++-aarch64-linux-gnu \
1111
g++-powerpc64le-linux-gnu \
1212
openjdk-8-jdk \
13+
pkg-config \
1314
&& \
1415
rm -rf /var/lib/apt/lists/*
16+
17+
RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
18+
tar xz -C /var/local
19+
ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:$PATH
20+

‎buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,11 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
99
curl \
1010
g++-s390x-linux-gnu \
1111
openjdk-8-jdk \
12+
pkg-config \
1213
&& \
1314
rm -rf /var/lib/apt/lists/*
15+
16+
RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
17+
tar xz -C /var/local
18+
ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:$PATH
19+

‎buildscripts/kokoro/android-interop.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ set -exu -o pipefail
77

88
cd github/grpc-java
99

10-
export LDFLAGS=-L/tmp/protobuf/lib
11-
export CXXFLAGS=-I/tmp/protobuf/include
10+
export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
11+
export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
1212
export LD_LIBRARY_PATH=/tmp/protobuf/lib
1313
export OS_NAME=$(uname)
1414

‎buildscripts/kokoro/android.sh

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ BASE_DIR="$(pwd)"
99

1010
cd "$BASE_DIR/github/grpc-java"
1111

12-
export LDFLAGS=-L/tmp/protobuf/lib
13-
export CXXFLAGS=-I/tmp/protobuf/include
14-
export LD_LIBRARY_PATH=/tmp/protobuf/lib
1512
export OS_NAME=$(uname)
1613

1714
cat <<EOF >> gradle.properties
@@ -30,10 +27,18 @@ unzip -qd "${ANDROID_HOME}/cmdline-tools" cmdline.zip
3027
rm cmdline.zip
3128
mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/latest"
3229
(yes || true) | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --licenses
33-
30+
curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
31+
tar xz -C /tmp
32+
export PATH=/tmp/cmake-3.26.3-linux-x86_64/bin:$PATH
33+
3434
# Proto deps
3535
buildscripts/make_dependencies.sh
3636

37+
sudo apt-get update && sudo apt-get install pkg-config
38+
export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
39+
export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
40+
export LD_LIBRARY_PATH=/tmp/protobuf/lib
41+
3742
# Build Android with Java 11, this adds it to the PATH
3843
sudo update-java-alternatives --set java-1.11.0-openjdk-amd64
3944
# Unset any existing JAVA_HOME env var to stop Gradle from using it
@@ -98,6 +103,7 @@ cd $BASE_DIR/github/grpc-java
98103
./gradlew clean
99104
git checkout HEAD^
100105
./gradlew --stop # use a new daemon to build the previous commit
106+
GRADLE_FLAGS="${GRADLE_FLAGS} -PskipCodegen=true" # skip codegen for build from previous commit since it wasn't built with --std=c++14 when making this change
101107
./gradlew publishToMavenLocal $GRADLE_FLAGS
102108
cd examples/android/helloworld/
103109
../../gradlew build $GRADLE_FLAGS

‎buildscripts/kokoro/unix.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@ fi
5151
export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
5252

5353
# Make protobuf discoverable by :grpc-compiler
54-
export LD_LIBRARY_PATH=/tmp/protobuf/lib
55-
export LDFLAGS=-L/tmp/protobuf/lib
56-
export CXXFLAGS="-I/tmp/protobuf/include"
54+
export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
55+
export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
56+
export LIBRARY_PATH=/tmp/protobuf/lib
5757

5858
./gradlew grpc-compiler:clean $GRADLE_FLAGS
5959

‎buildscripts/kokoro/windows.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
# Location of the continuous shell script in repository.
44
build_file: "grpc-java/buildscripts/kokoro/windows.bat"
5-
timeout_mins: 45
5+
timeout_mins: 90
66

77
# We always build mvn artifacts.
88
action {

‎buildscripts/kokoro/windows32.bat

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,21 @@ set ESCWORKSPACE=%WORKSPACE:\=\\%
1515

1616
@rem Clear JAVA_HOME to prevent a different Java version from being used
1717
set JAVA_HOME=
18-
set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH%
1918

2019
mkdir grpc-java-helper32
2120
cd grpc-java-helper32
22-
call "%VS140COMNTOOLS%\vsvars32.bat" || exit /b 1
21+
call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars32.bat" || exit /b 1
2322
call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1
2423

2524
cd "%WORKSPACE%"
2625

2726
SET TARGET_ARCH=x86_32
2827
SET FAIL_ON_WARNINGS=true
29-
SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\Release
30-
SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\include
28+
SET PROTOBUF_VER=22.5
29+
SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig
30+
SET VC_PROTOBUF_LIBS=/LIBPATH:%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib
31+
SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include
32+
call :Get_Libs
3133
SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true
3234
SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
3335

@@ -50,3 +52,32 @@ IF NOT %GRADLEEXIT% == 0 (
5052
cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
5153

5254
cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:publish" || exit /b 1
55+
56+
goto :eof
57+
:Get_Libs
58+
SetLocal EnableDelayedExpansion
59+
set "libs_list="
60+
for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do (
61+
for %%b in (%%a) do (
62+
set lib=%%b
63+
set libfirst2char=!lib:~0,2!
64+
if !libfirst2char!==-l (
65+
@rem remove the leading -l
66+
set lib=!lib:~2!
67+
@rem remove spaces
68+
set lib=!lib: =!
69+
@rem Because protobuf is specified as libprotobuf and elsewhere
70+
if !lib! NEQ protobuf (
71+
set lib=!lib!.lib
72+
if "!libs_list!"=="" (
73+
set libs_list=!lib!
74+
) else (
75+
set libs_list=!libs_list!,!lib!
76+
)
77+
)
78+
)
79+
)
80+
)
81+
EndLocal & set "VC_PROTOBUF_LIBS=%VC_PROTOBUF_LIBS%,%libs_list%"
82+
exit /b 0
83+

‎buildscripts/kokoro/windows64.bat

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,54 @@ set ESCWORKSPACE=%WORKSPACE:\=\\%
1414

1515
@rem Clear JAVA_HOME to prevent a different Java version from being used
1616
set JAVA_HOME=
17-
set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH%
1817

1918
mkdir grpc-java-helper64
2019
cd grpc-java-helper64
21-
call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" || exit /b 1
20+
call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1
2221
call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1
2322

2423
cd "%WORKSPACE%"
2524

2625
SET TARGET_ARCH=x86_64
2726
SET FAIL_ON_WARNINGS=true
28-
SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\Release
29-
SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\include
27+
SET PROTOBUF_VER=22.5
28+
SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig
29+
SET VC_PROTOBUF_LIBS=/LIBPATH:%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib
30+
SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include
31+
call :Get_Libs
3032
SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true
3133
SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
3234

3335
@rem make sure no daemons have any files open
3436
cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
3537

3638
cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts grpc-compiler:clean grpc-compiler:build grpc-compiler:publish" || exit /b 1
39+
40+
goto :eof
41+
:Get_Libs
42+
SetLocal EnableDelayedExpansion
43+
set "libs_list="
44+
for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do (
45+
for %%b in (%%a) do (
46+
set lib=%%b
47+
set libfirst2char=!lib:~0,2!
48+
if !libfirst2char!==-l (
49+
@rem remove the leading -l
50+
set lib=!lib:~2!
51+
@rem remove spaces
52+
set lib=!lib: =!
53+
@rem Because protobuf is specified as libprotobuf and elsewhere
54+
if !lib! NEQ protobuf (
55+
set lib=!lib!.lib
56+
if "!libs_list!"=="" (
57+
set libs_list=!lib!
58+
) else (
59+
set libs_list=!libs_list!,!lib!
60+
)
61+
)
62+
)
63+
)
64+
)
65+
EndLocal & set "VC_PROTOBUF_LIBS=%VC_PROTOBUF_LIBS%,%libs_list%"
66+
exit /b 0
67+

‎buildscripts/make_dependencies.bat

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
1-
set PROTOBUF_VER=21.7
2-
set CMAKE_NAME=cmake-3.3.2-win32-x86
1+
choco install -y pkgconfiglite
2+
choco install -y openjdk --version=17.0
3+
set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-17\bin"
4+
set PROTOBUF_VER=22.5
5+
set ABSL_VERSION=20230125.4
6+
set CMAKE_NAME=cmake-3.26.3-windows-x86_64
37

48
if not exist "protobuf-%PROTOBUF_VER%\build\Release\" (
59
call :installProto || exit /b 1
610
)
711

812
echo Compile gRPC-Java with something like:
9-
echo -PtargetArch=x86_32 -PvcProtobufLibs=%cd%\protobuf-%PROTOBUF_VER%\build\Release -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\include
13+
echo -PtargetArch=x86_32 -PvcProtobufLibPath=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\lib -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\include -PvcProtobufLibs=insert-list-of-libs-from-pkg-config-output-here
1014
goto :eof
1115

1216

@@ -20,25 +24,35 @@ if not exist "%CMAKE_NAME%" (
2024
set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin
2125
:hasCmake
2226
@rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled
23-
powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
27+
powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
2428
powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1
2529
del protobuf.zip
30+
powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1
31+
powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1
32+
del absl.zip
33+
rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
34+
move abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
2635
mkdir protobuf-%PROTOBUF_VER%\build
2736
pushd protobuf-%PROTOBUF_VER%\build
2837

29-
@rem Workaround https://github.com/protocolbuffers/protobuf/issues/10174
30-
powershell -command "(Get-Content ..\cmake\extract_includes.bat.in) -replace '\.\.\\', '' | Out-File -encoding ascii ..\cmake\extract_includes.bat.in"
3138
@rem cmake does not detect x86_64 from the vcvars64.bat variables.
32-
@rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS
33-
if "%PLATFORM%" == "X64" (
34-
@rem Note the space
35-
SET CMAKE_VSARCH= Win64
39+
@rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS, likewise for x32
40+
if "%PLATFORM%" == "x64" (
41+
SET CMAKE_VSARCH=-A x64
42+
) else if "%PLATFORM%" == "x86" (
43+
@rem -A x86 doesn't work: https://github.com/microsoft/vcpkg/issues/15465
44+
SET CMAKE_VSARCH=-DCMAKE_GENERATOR_PLATFORM=WIN32
3645
) else (
3746
SET CMAKE_VSARCH=
3847
)
39-
cmake -Dprotobuf_BUILD_TESTS=OFF -G "Visual Studio %VisualStudioVersion:~0,2%%CMAKE_VSARCH%" .. || exit /b 1
40-
msbuild /maxcpucount /p:Configuration=Release /verbosity:minimal libprotoc.vcxproj || exit /b 1
41-
call extract_includes.bat || exit /b 1
48+
for /f "tokens=4 delims=\" %%a in ("%VCINSTALLDIR%") do (
49+
SET VC_YEAR=%%a
50+
)
51+
for /f "tokens=1 delims=." %%a in ("%VisualStudioVersion%") do (
52+
SET visual_studio_major_version=%%a
53+
)
54+
cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%cd%\protobuf-%PROTOBUF_VER% -DCMAKE_PREFIX_PATH=%cd%\protobuf-%PROTOBUF_VER% -G "Visual Studio %visual_studio_major_version% %VC_YEAR%" %CMAKE_VSARCH% .. || exit /b 1
55+
cmake --build . --config Release --target install || exit /b 1
4256
popd
4357
goto :eof
4458

@@ -49,3 +63,4 @@ powershell -command "$ErrorActionPreference = 'stop'; & { iwr https://cmake.org/
4963
powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('cmake.zip', '.') }" || exit /b 1
5064
del cmake.zip
5165
goto :eof
66+

0 commit comments

Comments
 (0)