Skip to content

Upgrade Protobuf C++ to 22.3 #10048

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions COMPILING.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code
generation. Most users only need to use `skipCodegen=true` as discussed above.

### Build Protobuf
The codegen plugin is C++ code and requires protobuf 21.7 or later.
The codegen plugin is C++ code and requires protobuf 22.3 or later.

For Linux, Mac and MinGW:
```
$ PROTOBUF_VERSION=21.7
$ PROTOBUF_VERSION=22.3
$ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz
$ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz
$ cd protobuf-$PROTOBUF_VERSION
Expand Down
14 changes: 7 additions & 7 deletions buildscripts/grpc-java-artifacts/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
FROM centos:7.9.2009

RUN yum install -y \
autoconf \
automake \
gcc-c++ \
gcc-c++.i686 \
RUN yum install -y centos-release-scl && \
yum install -y \
devtoolset-7-gcc-c++ \
glibc-devel \
glibc-devel.i686 \
java-1.8.0-openjdk-devel \
Expand All @@ -19,7 +17,9 @@ RUN yum install -y \
&& \
yum clean all

# Install Maven
RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
tar xz -C /var/local
RUN curl -Ls http://dlcdn.apache.org/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | \
tar xz -C /var/local
ENV PATH /var/local/apache-maven-3.3.9/bin:$PATH

ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.3.9/bin:$PATH
1 change: 1 addition & 0 deletions buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
cmake \
build-essential \
ca-certificates \
curl \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
cmake \
build-essential \
ca-certificates \
curl \
Expand Down
2 changes: 1 addition & 1 deletion buildscripts/kokoro/linux_artifacts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)"
trap spongify_logs EXIT

"$GRPC_JAVA_DIR"/buildscripts/build_docker.sh
"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-x86 /grpc-java/buildscripts/build_artifacts_in_docker.sh
"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-x86 scl enable devtoolset-7 /grpc-java/buildscripts/build_artifacts_in_docker.sh

# grpc-android, grpc-cronet and grpc-binder require the Android SDK, so build outside of Docker and
# use --include-build for its grpc-core dependency
Expand Down
11 changes: 9 additions & 2 deletions buildscripts/make_dependencies.bat
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
set PROTOBUF_VER=21.7
set CMAKE_NAME=cmake-3.3.2-win32-x86
@rem set PROTOBUF_VER=22.3
set ABSL_VERSION=20230125.2
set CMAKE_NAME=cmake-3.26.3-windows-x86_64

if not exist "protobuf-%PROTOBUF_VER%\build\Release\" (
call :installProto || exit /b 1
Expand All @@ -20,9 +22,14 @@ if not exist "%CMAKE_NAME%" (
set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin
:hasCmake
@rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled
powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1
del protobuf.zip
powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1
powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1
del absl.zip
@rem rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
@rem rename abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
mkdir protobuf-%PROTOBUF_VER%\build
pushd protobuf-%PROTOBUF_VER%\build

Expand Down
54 changes: 36 additions & 18 deletions buildscripts/make_dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@
# Build protoc
set -evux -o pipefail

PROTOBUF_VERSION=21.7
PROTOBUF_VERSION=22.3
ABSL_VERSION=20230125.2

# ARCH is x86_64 bit unless otherwise specified.
ARCH="${ARCH:-x86_64}"
DOWNLOAD_DIR=/tmp/source
INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH"
BUILDSCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
mkdir -p $DOWNLOAD_DIR
cd "$DOWNLOAD_DIR"

# Start with a sane default
NUM_CPU=4
Expand All @@ -26,27 +29,42 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then
echo "Not building protobuf. Already built"
# TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date
else
if [[ ! -d "$DOWNLOAD_DIR"/protobuf-"${PROTOBUF_VERSION}" ]]; then
curl -Ls https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR
if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then
curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz
curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz
rmdir "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp"
mv "abseil-cpp-$ABSL_VERSION" "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp"
fi
pushd $DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}
# the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first
rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
mkdir "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
pushd "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
# install here so we don't need sudo
if [[ "$ARCH" == x86* ]]; then
./configure CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} --disable-shared \
--prefix="$INSTALL_DIR"
elif [[ "$ARCH" == aarch* ]]; then
./configure --disable-shared --host=aarch64-linux-gnu --prefix="$INSTALL_DIR"
elif [[ "$ARCH" == ppc* ]]; then
./configure --disable-shared --host=powerpc64le-linux-gnu --prefix="$INSTALL_DIR"
elif [[ "$ARCH" == s390* ]]; then
./configure --disable-shared --host=s390x-linux-gnu --prefix="$INSTALL_DIR"
elif [[ "$ARCH" == loongarch* ]]; then
./configure --disable-shared --host=loongarch64-unknown-linux-gnu --prefix="$INSTALL_DIR"
CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \
-DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \
-DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
else
if [[ "$ARCH" == aarch_64 ]]; then
GCC_ARCH=aarch64-linux-gnu
elif [[ "$ARCH" == ppcle_64 ]]; then
GCC_ARCH=powerpc64le-linux-gnu
elif [[ "$ARCH" == s390_64 ]]; then
GCC_ARCH=s390x-linux-gnu
elif [[ "$ARCH" == loongarch_64 ]]; then
GCC_ARCH=loongarch64-unknown-linux-gnu
else
echo "Unknown architecture: $ARCH"
exit 1
fi
cmake .. \
-DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \
-DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \
-DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake
fi
# the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first
make clean
make V=0 -j$NUM_CPU
make install
cmake --build . -j "$NUM_CPU"
cmake --install .
[ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib"
popd
fi

Expand Down
9 changes: 9 additions & 0 deletions buildscripts/toolchain.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
set(CMAKE_SYSTEM_NAME Linux)

set(CMAKE_C_COMPILER "${crosscompile_ARCH}-gcc")
set(CMAKE_CXX_COMPILER "${crosscompile_ARCH}-g++")
set(CMAKE_FIND_ROOT_PATH "/usr/${crosscompile_ARCH}/")

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
2 changes: 1 addition & 1 deletion compiler/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ model {
all {
if (toolChain in Gcc || toolChain in Clang) {
cppCompiler.define("GRPC_VERSION", version)
cppCompiler.args "--std=c++0x"
cppCompiler.args "--std=c++14"
addEnvArgs("CXXFLAGS", cppCompiler.args)
addEnvArgs("CPPFLAGS", cppCompiler.args)
if (osdetector.os == "osx") {
Expand Down