Skip to content

Commit d14aada

Browse files
merlimatmerlimat
authored
Add CI job to build and run Python tests (#9)
Fixed build and added Github actions to validate PR with unit tests
1 parent 9f6fc69 commit d14aada

34 files changed

+1477
-72
lines changed

‎.github/workflows/ci-pr-validation.yaml

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: PR validation
21+
on:
22+
pull_request:
23+
branches: ['main']
24+
25+
concurrency:
26+
group: ${{ github.workflow }}-${{ github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
31+
unit-tests:
32+
name: Run unit tests
33+
runs-on: ubuntu-latest
34+
timeout-minutes: 120
35+
36+
steps:
37+
- name: checkout
38+
uses: actions/checkout@v3
39+
40+
- name: Install deps
41+
run: sudo apt-get install -y libboost-python-dev
42+
43+
- name: Install Pulsar C++ client
44+
run: build-support/install-cpp-client.sh
45+
46+
- name: CMake
47+
run: cmake .
48+
49+
- name: Build
50+
run: make -j8
51+
52+
- name: Python install
53+
run: |
54+
./setup.py bdist_wheel
55+
WHEEL=$(find dist -name '*.whl')
56+
pip3 install ${WHEEL}[avro]
57+
58+
- name: Start Pulsar service
59+
run: ./build-support/pulsar-test-service-start.sh
60+
61+
- name: Run unit tests
62+
run: ./tests/run-unit-tests.sh
63+
64+
- name: Stop Pulsar service
65+
run: ./build-support/pulsar-test-service-stop.sh
66+

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,6 @@ Makefile
99
_pulsar.so
1010
cmake_install.cmake
1111
__pycache__
12+
.build
13+
.pulsar-mac-wheels-cache
14+
.DS_Store

‎CMakeLists.txt

+20-42
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,26 @@ project (pulsar-client-python)
2121
cmake_minimum_required(VERSION 3.12)
2222
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
2323

24+
option(LINK_STATIC "Link against static libraries" OFF)
25+
MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC})
26+
2427
MESSAGE(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE})
2528
set(THREADS_PREFER_PTHREAD_FLAG TRUE)
2629
find_package(Threads REQUIRED)
2730
MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT})
2831

29-
30-
find_library(PULSAR_LIBRARY NAMES libpulsar.a)
32+
if (LINK_STATIC)
33+
find_library(PULSAR_LIBRARY NAMES libpulsar.a)
34+
else()
35+
find_library(PULSAR_LIBRARY NAMES libpulsar.so)
36+
endif()
3137
message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}")
3238

3339
find_path(PULSAR_INCLUDE pulsar/Client.h)
3440
message(STATUS "PULSAR_INCLUDE: ${PULSAR_INCLUDE}")
3541

3642
SET(Boost_NO_BOOST_CMAKE ON)
37-
SET(Boost_USE_STATIC_LIBS ON)
43+
SET(Boost_USE_STATIC_LIBS ${LINK_STATIC})
3844

3945
SET(CMAKE_CXX_STANDARD 11)
4046

@@ -63,32 +69,6 @@ endif ()
6369

6470
MESSAGE(STATUS "BOOST_PYTHON_NAME_FOUND: " ${BOOST_PYTHON_NAME_FOUND})
6571

66-
set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/)
67-
68-
### This part is to find and keep SSL dynamic libs in RECORD_OPENSSL_SSL_LIBRARY and RECORD_OPENSSL_CRYPTO_LIBRARY
69-
### After find the libs, will unset related cache, and will not affect another same call to find_package.
70-
if (APPLE)
71-
set(OPENSSL_INCLUDE_DIR /usr/local/opt/openssl/include/ /opt/homebrew/opt/openssl/include)
72-
set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl)
73-
endif ()
74-
75-
set(OPENSSL_USE_STATIC_LIBS TRUE)
76-
find_package(OpenSSL REQUIRED)
77-
78-
find_library(ZLIB_LIBRARIES REQUIRED NAMES libz.a z zlib)
79-
message(STATUS "ZLIB_LIBRARIES: ${ZLIB_LIBRARIES}")
80-
81-
find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
82-
message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
83-
find_library(Protobuf_LIBRARIES NAMES libprotobuf.a libprotobuf)
84-
message(STATUS "Protobuf: ${Protobuf_LIBRARIES}")
85-
find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
86-
message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
87-
find_library(LIB_ZSTD NAMES libzstd.a)
88-
message(STATUS "ZStd: ${LIB_ZSTD}")
89-
find_library(LIB_SNAPPY NAMES libsnappy.a)
90-
message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}")
91-
9272
########################################################################################################################
9373

9474
INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} "${Boost_INCLUDE_DIRS}" "${Python3_INCLUDE_DIRS}")
@@ -138,12 +118,6 @@ endif()
138118

139119
# Try all possible boost-python variable namings
140120
set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY}
141-
${OPENSSL_LIBRARIES}
142-
${ZLIB_LIBRARIES}
143-
${CURL_LIBRARIES}
144-
${Protobuf_LIBRARIES}
145-
${LIB_ZSTD}
146-
${LIB_SNAPPY}
147121
${Boost_PYTHON_LIBRARY}
148122
${Boost_PYTHON3_LIBRARY}
149123
${Boost_PYTHON37-MT_LIBRARY}
@@ -173,13 +147,17 @@ endif ()
173147

174148
message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}")
175149

176-
if (APPLE)
177-
set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
178-
target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS})
179-
else ()
180-
if (NOT MSVC)
181-
set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++")
182-
endif()
150+
if (LINK_STATIC)
151+
if (APPLE)
152+
set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
153+
target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS})
154+
else ()
155+
if (NOT MSVC)
156+
set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++")
157+
endif()
158+
target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
159+
endif ()
160+
else()
183161
target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
184162
endif ()
185163

‎build-support/install-cpp-client.sh

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
set -e -x
22+
23+
ROOT_DIR=$(git rev-parse --show-toplevel)
24+
25+
cd $ROOT_DIR
26+
27+
CPP_CLIENT_VERSION=$(cat pulsar-client-cpp-version.txt | xargs)
28+
29+
# Fetch the client binaries
30+
## TODO: Fetch from official release once it's available
31+
pushd /tmp
32+
curl -L -O https://github.com/merlimat/pulsar-client-cpp/releases/download/${CPP_CLIENT_VERSION}/apache-pulsar-client.deb
33+
curl -L -O https://github.com/merlimat/pulsar-client-cpp/releases/download/${CPP_CLIENT_VERSION}/apache-pulsar-client-dev.deb
34+
popd
35+
36+
sudo apt install /tmp/apache-pulsar-client.deb /tmp/apache-pulsar-client-dev.deb

‎build-support/pulsar-test-service-start.sh

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
set -e
22+
23+
SRC_DIR=$(git rev-parse --show-toplevel)
24+
cd $SRC_DIR
25+
26+
./build-support/pulsar-test-service-stop.sh
27+
28+
CONTAINER_ID=$(docker run -i -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600)
29+
echo $CONTAINER_ID > .tests-container-id.txt
30+
31+
docker cp tests/test-conf $CONTAINER_ID:/pulsar/test-conf
32+
docker cp build-support/start-test-service-inside-container.sh $CONTAINER_ID:start-test-service-inside-container.sh
33+
34+
docker exec -i $CONTAINER_ID /start-test-service-inside-container.sh
35+
36+
docker cp $CONTAINER_ID:/pulsar/data/tokens/token.txt tests/.test-token.txt
37+
38+
echo "-- Ready to start tests"

‎build-support/pulsar-test-service-stop.sh

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
set -e
22+
23+
SRC_DIR=$(git rev-parse --show-toplevel)
24+
cd $SRC_DIR
25+
26+
CONTAINER_ID_PATH=".tests-container-id.txt"
27+
28+
if [ -f ${CONTAINER_ID_PATH} ]; then
29+
CONTAINER_ID=$(cat $CONTAINER_ID_PATH)
30+
docker kill $CONTAINER_ID || true
31+
rm .tests-container-id.txt
32+
fi

‎build-support/start-test-service-inside-container.sh

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
set -e -x
22+
23+
export PULSAR_EXTRA_OPTS=-Dpulsar.auth.basic.conf=test-conf/.htpasswd
24+
25+
# Generate secret key and token
26+
mkdir -p data/tokens
27+
bin/pulsar tokens create-secret-key --output data/tokens/secret.key
28+
29+
bin/pulsar tokens create \
30+
--subject token-principal \
31+
--secret-key file:///pulsar/data/tokens/secret.key \
32+
> /pulsar/data/tokens/token.txt
33+
34+
export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl.conf
35+
export PULSAR_PID_DIR=/tmp
36+
bin/pulsar-daemon start standalone \
37+
--no-functions-worker --no-stream-storage \
38+
--bookkeeper-dir data/bookkeeper
39+
40+
echo "-- Wait for Pulsar service to be ready"
41+
until curl http://localhost:8080/metrics > /dev/null 2>&1 ; do sleep 1; done
42+
43+
echo "-- Pulsar service is ready -- Configure permissions"
44+
45+
export PULSAR_CLIENT_CONF=test-conf/client-ssl.conf
46+
47+
# Create "standalone" cluster if it does not exist
48+
bin/pulsar-admin clusters list | grep -q '^standalone$' ||
49+
bin/pulsar-admin clusters create \
50+
standalone \
51+
--url http://localhost:8080/ \
52+
--url-secure https://localhost:8443/ \
53+
--broker-url pulsar://localhost:6650/ \
54+
--broker-url-secure pulsar+ssl://localhost:6651/
55+
56+
# Update "public" tenant
57+
bin/pulsar-admin tenants create public -r "anonymous" -c "standalone"
58+
59+
# Update "public/default" with no auth required
60+
bin/pulsar-admin namespaces create public/default -c standalone
61+
bin/pulsar-admin namespaces grant-permission public/default \
62+
--actions produce,consume \
63+
--role "anonymous"
64+
65+
# Create "public/default-2" with no auth required
66+
bin/pulsar-admin namespaces create public/default-2 \
67+
--clusters standalone
68+
bin/pulsar-admin namespaces grant-permission public/default-2 \
69+
--actions produce,consume \
70+
--role "anonymous"
71+
72+
# Create "public/default-3" with no auth required
73+
bin/pulsar-admin namespaces create public/default-3 \
74+
--clusters standalone
75+
bin/pulsar-admin namespaces grant-permission public/default-3 \
76+
--actions produce,consume \
77+
--role "anonymous"
78+
79+
# Create "public/default-4" with encryption required
80+
bin/pulsar-admin namespaces create public/default-4 \
81+
--clusters standalone
82+
bin/pulsar-admin namespaces grant-permission public/default-4 \
83+
--actions produce,consume \
84+
--role "anonymous"
85+
bin/pulsar-admin namespaces set-encryption-required public/default-4 -e
86+
87+
# Create "public/test-backlog-quotas" to test backlog quotas policy
88+
bin/pulsar-admin namespaces create public/test-backlog-quotas \
89+
--clusters standalone
90+
91+
# Create "private" tenant
92+
bin/pulsar-admin tenants create private -r "" -c "standalone"
93+
94+
# Create "private/auth" with required authentication
95+
bin/pulsar-admin namespaces create private/auth --clusters standalone
96+
97+
bin/pulsar-admin namespaces grant-permission private/auth \
98+
--actions produce,consume \
99+
--role "token-principal"
100+
101+
echo "-- Ready to start tests"

‎test_consumer.py renamed to ‎examples/consumer.py

File renamed without changes.

‎test_producer.py renamed to ‎examples/producer.py

File renamed without changes.

‎pulsar-client-cpp-version.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.0.0-pre-1

0 commit comments

Comments
 (0)