Skip to content

Commit 5eefb7f

Browse files
committed
Fix Protobuf symbols not found in libpulsarwithdeps.a when building on macOS
### Motivation #290 brings a regression that on macOS, Protobuf is always found with CMake Config mode, which does not set the `Protobuf_LIBRARIES` variable so that the libpulsarwithdeps.a misses the symbols of Protobuf. ### Modifications When `LINK_STATIC` is ON, use CMake Module mode to find the Protobuf. Add `build-static-library.sh` to build libraries with static dependencies and verify these libraries in PR workflow. Upload the pre-built binaries in the build workflow.
1 parent 37ea769 commit 5eefb7f

File tree

6 files changed

+251
-5
lines changed

6 files changed

+251
-5
lines changed

.github/workflows/ci-build-binary-artifacts.yaml

+33
Original file line numberDiff line numberDiff line change
@@ -192,3 +192,36 @@ jobs:
192192
with:
193193
name: ${{ matrix.triplet }}-Debug
194194
path: ${{ env.INSTALL_DIR }}-Debug
195+
196+
package-macos:
197+
name: Build macOS libraries
198+
runs-on: macos-latest
199+
timeout-minutes: 500
200+
201+
strategy:
202+
fail-fast: true
203+
matrix:
204+
arch:
205+
- x86_64
206+
- arm64
207+
208+
steps:
209+
- name: checkout
210+
uses: actions/checkout@v3
211+
212+
- name: Install dependencies
213+
run: |
214+
export ARCH=${{ matrix.arch }}
215+
./pkg/mac/build-static-library.sh
216+
217+
- name: Zip artifact
218+
run: |
219+
cd ./pkg/mac/.install
220+
zip -r macos-${{matrix.arch}}.zip ./include/pulsar/* ./lib/*
221+
cp macos-${matrix.arch}.zip ../../../
222+
223+
- name: Upload artifacts
224+
uses: actions/upload-artifact@v3
225+
with:
226+
name: macos-${matrix.arch}.zip
227+
path: macos-${matrix.arch}.zip

.github/workflows/ci-pr-validation.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,28 @@ jobs:
302302
run: |
303303
cmake --build ./build-macos --parallel --config Release
304304
305+
cpp-build-macos-static:
306+
timeout-minutes: 120
307+
name: Build CPP Client on macOS with static dependencies
308+
runs-on: macos-12
309+
needs: unit-tests
310+
steps:
311+
- name: checkout
312+
uses: actions/checkout@v3
313+
314+
- name: Build libraries
315+
run: ./pkg/mac/build-static-library.sh
316+
317+
- name: Test static libraries
318+
run: |
319+
export PULSAR_DIR=$PWD/pkg/mac/.install
320+
echo "Build with static library"
321+
clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a
322+
./static.out
323+
echo "Build with dynamic library"
324+
clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR -lpulsar
325+
./dynamic.out
326+
305327
# Job that will be required to complete and depends on all the other jobs
306328
check-completion:
307329
name: Check Completion

.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -104,3 +104,5 @@ vcpkg_installed/
104104
.tests-container-id.txt
105105
Testing
106106
.test-token.txt
107+
pkg/mac/.build
108+
pkg/mac/.install

CMakeLists.txt

+12-4
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,17 @@ find_package(Threads REQUIRED)
8888
MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT})
8989

9090
set(Boost_NO_BOOST_CMAKE ON)
91+
92+
if (APPLE AND NOT LINK_STATIC)
93+
# The latest Protobuf dependency on macOS requires the C++17 support and
94+
# it could only be found by the CONFIG mode
95+
set(LATEST_PROTOBUF TRUE)
96+
else ()
97+
set(LATEST_PROTOBUF FALSE)
98+
endif ()
99+
91100
if (NOT CMAKE_CXX_STANDARD)
92-
if (APPLE)
93-
# The latest Protobuf dependency on macOS requires the C++17 support
101+
if (LATEST_PROTOBUF)
94102
set(CMAKE_CXX_STANDARD 17)
95103
else ()
96104
set(CMAKE_CXX_STANDARD 11)
@@ -143,7 +151,7 @@ find_package(OpenSSL REQUIRED)
143151
message("OPENSSL_INCLUDE_DIR: " ${OPENSSL_INCLUDE_DIR})
144152
message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES})
145153

146-
if (APPLE)
154+
if (LATEST_PROTOBUF)
147155
# See https://github.com/apache/arrow/issues/35987
148156
add_definitions(-DPROTOBUF_USE_DLLS)
149157
# Use Config mode to avoid FindProtobuf.cmake does not find the Abseil library
@@ -318,7 +326,7 @@ set(COMMON_LIBS
318326
${CMAKE_DL_LIBS}
319327
)
320328

321-
if (APPLE)
329+
if (LATEST_PROTOBUF)
322330
# Protobuf_LIBRARIES is empty when finding Protobuf in Config mode
323331
set(COMMON_LIBS ${COMMON_LIBS} protobuf::libprotobuf)
324332
else ()

dependencies.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,5 @@ protobuf: 3.20.0
2323
zlib: 1.2.12
2424
zstd: 1.5.2
2525
snappy: 1.1.9
26-
openssl: 1.1.1q
26+
openssl: 1.1.1v
2727
curl: 8.4.0

pkg/mac/build-static-library.sh

+181
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
#!/bin/bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
set -ex
22+
cd `dirname $0`
23+
24+
pip3 install pyyaml
25+
26+
MACOSX_DEPLOYMENT_TARGET=10.15
27+
if [[ -z ${ARCH} ]]; then
28+
ARCH=`uname -m`
29+
fi
30+
31+
BUILD_DIR=$PWD/.build
32+
INSTALL_DIR=$PWD/.install
33+
PREFIX=$BUILD_DIR/install
34+
mkdir -p $BUILD_DIR
35+
cp -f ../../build-support/dep-version.py $BUILD_DIR/
36+
cp -f ../../dependencies.yaml $BUILD_DIR/
37+
38+
pushd $BUILD_DIR
39+
40+
BOOST_VERSION=$(./dep-version.py boost)
41+
ZLIB_VERSION=$(./dep-version.py zlib)
42+
OPENSSL_VERSION=$(./dep-version.py openssl)
43+
PROTOBUF_VERSION=$(./dep-version.py protobuf)
44+
ZSTD_VERSION=$(./dep-version.py zstd)
45+
SNAPPY_VERSION=$(./dep-version.py snappy)
46+
CURL_VERSION=$(./dep-version.py curl)
47+
48+
BOOST_VERSION_=${BOOST_VERSION//./_}
49+
if [ ! -f boost/.done ]; then
50+
echo "Building Boost $BOOST_VERSION"
51+
curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz
52+
tar zxf boost_${BOOST_VERSION_}.tar.gz
53+
mkdir -p $PREFIX/include
54+
cp -rf boost_${BOOST_VERSION_}/boost $PREFIX/include/
55+
mkdir -p boost
56+
touch boost/.done
57+
else
58+
echo "Using cached Boost $BOOST_VERSION"
59+
fi
60+
61+
if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then
62+
echo "Building ZLib $ZLIB_VERSION"
63+
curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz
64+
tar zxf zlib-${ZLIB_VERSION}.tar.gz
65+
pushd zlib-$ZLIB_VERSION
66+
CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
67+
make -j16
68+
make install
69+
touch .done
70+
popd
71+
else
72+
echo "Using cached ZLib $ZLIB_VERSION"
73+
fi
74+
75+
OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
76+
if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then
77+
echo "Building OpenSSL $OPENSSL_VERSION"
78+
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz
79+
tar zxf OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz
80+
81+
pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
82+
if [[ $ARCH = 'arm64' ]]; then
83+
PLATFORM=darwin64-arm64-cc
84+
else
85+
PLATFORM=darwin64-x86_64-cc
86+
fi
87+
CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
88+
./Configure --prefix=$PREFIX no-shared no-unit-test $PLATFORM
89+
make -j8 >/dev/null
90+
make install_sw >/dev/null
91+
popd
92+
93+
touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done
94+
else
95+
echo "Using cached OpenSSL $OPENSSL_VERSION"
96+
fi
97+
98+
if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then
99+
echo "Building Protobuf $PROTOBUF_VERSION"
100+
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
101+
tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
102+
pushd protobuf-${PROTOBUF_VERSION}
103+
pushd cmake/
104+
cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
105+
-Dprotobuf_BUILD_TESTS=OFF \
106+
-DCMAKE_INSTALL_PREFIX=$PREFIX
107+
cmake --build build -j16 --target install
108+
popd
109+
touch .done
110+
popd
111+
else
112+
echo "Using cached Protobuf $PROTOBUF_VERSION"
113+
fi
114+
115+
if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then
116+
echo "Building ZStd $ZSTD_VERSION"
117+
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
118+
tar zxf zstd-${ZSTD_VERSION}.tar.gz
119+
pushd zstd-${ZSTD_VERSION}
120+
CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \
121+
make -j16 -C lib install-static install-includes
122+
touch .done
123+
popd
124+
else
125+
echo "Using cached ZStd $ZSTD_VERSION"
126+
fi
127+
128+
if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then
129+
echo "Building Snappy $SNAPPY_VERSION"
130+
curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz
131+
tar zxf ${SNAPPY_VERSION}.tar.gz
132+
pushd snappy-${SNAPPY_VERSION}
133+
CXXFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
134+
cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF
135+
make -j16
136+
make install
137+
touch .done
138+
popd
139+
else
140+
echo "Using cached Snappy $SNAPPY_VERSION"
141+
fi
142+
143+
if [ ! -f curl-${CURL_VERSION}/.done ]; then
144+
echo "Building LibCurl $CURL_VERSION"
145+
CURL_VERSION_=${CURL_VERSION//./_}
146+
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz
147+
tar zxf curl-${CURL_VERSION}.tar.gz
148+
pushd curl-${CURL_VERSION}
149+
# Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl.
150+
cp -rf $PREFIX/include/openssl include/
151+
CFLAGS="-I$PREFIX/include -fPIC -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
152+
./configure --with-ssl=$PREFIX \
153+
--without-nghttp2 \
154+
--without-libidn2 \
155+
--disable-ldap \
156+
--without-brotli \
157+
--without-secure-transport \
158+
--disable-ipv6 \
159+
--prefix=$PREFIX
160+
make -j16 install
161+
touch .done
162+
popd
163+
else
164+
echo "Using cached LibCurl $CURL_VERSION"
165+
fi
166+
167+
popd # pkg/mac
168+
cd ../../ # project root
169+
170+
cmake -B build-static -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \
171+
-DLINK_STATIC=ON \
172+
-DBUILD_TESTS=OFF \
173+
-DBUILD_DYNAMIC_LIB=ON \
174+
-DBUILD_STATIC_LIB=ON \
175+
-DCMAKE_OSX_ARCHITECTURES="${ARCH}" \
176+
-DCMAKE_PREFIX_PATH=$PREFIX \
177+
-DOPENSSL_ROOT_DIR=$PREFIX \
178+
-DPROTOC_PATH=$PREFIX/bin/protoc \
179+
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
180+
-DCMAKE_BUILD_TYPE=Release
181+
cmake --build build-static -j16 --target install

0 commit comments

Comments
 (0)