diff --git a/build-support/copy-deps-versionfile.sh b/build-support/copy-deps-versionfile.sh index 0a36d10..3e42105 100755 --- a/build-support/copy-deps-versionfile.sh +++ b/build-support/copy-deps-versionfile.sh @@ -25,5 +25,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel) for dir in manylinux2014 manylinux_musl; do mkdir -p pkg/$dir/.build cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build + cp $ROOT_DIR/pulsar-client-cpp-version.txt pkg/$dir/.build cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build done diff --git a/dependencies.yaml b/dependencies.yaml index 262b100..3d76f96 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -18,8 +18,6 @@ # boost: 1.80.0 - -# Only used for MacOS builds cmake: 3.24.2 protobuf: 3.20.0 zlib: 1.2.13 diff --git a/pkg/build-wheel-inside-docker.sh b/pkg/build-wheel-inside-docker.sh index a477b35..09b4304 100755 --- a/pkg/build-wheel-inside-docker.sh +++ b/pkg/build-wheel-inside-docker.sh @@ -22,8 +22,6 @@ set -e -x cd /pulsar-client-python -build-support/install-cpp-client.sh - rm -f CMakeCache.txt CMakeFiles cmake . \ diff --git a/pkg/manylinux2014/Dockerfile b/pkg/manylinux2014/Dockerfile index 7355585..70f5e31 100644 --- a/pkg/manylinux2014/Dockerfile +++ b/pkg/manylinux2014/Dockerfile @@ -22,7 +22,6 @@ FROM quay.io/pypa/manylinux2014_${ARCH} ARG PYTHON_VERSION ARG PYTHON_SPEC -ARG PLATFORM ARG ARCH ENV PYTHON_VERSION=${PYTHON_VERSION} @@ -36,6 +35,7 @@ ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} RUN pip3 install pyyaml ADD .build/dependencies.yaml / +ADD .build/pulsar-client-cpp-version.txt / ADD .build/dep-version.py /usr/local/bin # Download and compile boost @@ -54,3 +54,71 @@ RUN CMAKE_VERSION=$(dep-version.py cmake) && \ cp cmake-${CMAKE_VERSION}-linux-${ARCH}/bin/* /usr/bin/ && \ cp -r cmake-${CMAKE_VERSION}-linux-${ARCH}/share/cmake-* /usr/share/ && \ rm -rf cmake-${CMAKE_VERSION}-linux-${ARCH} cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz + +# Download and compile protobuf +RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + cd protobuf-${PROTOBUF_VERSION}/ && \ + CXXFLAGS=-fPIC ./configure && \ + make -j8 && make install && ldconfig && \ + rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} + +# ZLib +RUN ZLIB_VERSION=$(dep-version.py zlib) && \ + curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ + tar xfz v${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ + CFLAGS="-fPIC -O3" ./configure && \ + make -j8 && make install && \ + rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} + +# Zstandard +RUN ZSTD_VERSION=$(dep-version.py zstd) && \ + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ + tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ + cd zstd-${ZSTD_VERSION} && \ + CFLAGS="-fPIC -O3" make -j8 && \ + make install && \ + rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz + +# Snappy +RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ + curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ + tar xfz ${SNAPPY_VERSION}.tar.gz && \ + cd snappy-${SNAPPY_VERSION} && \ + CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ + make -j8 && make install && \ + rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz + +RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ + OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ + ./config -fPIC --prefix=/usr/local/ssl/ && \ + make -j8 && make install && \ + rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} + +ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ + +# LibCurl +RUN CURL_VERSION=$(dep-version.py curl) && \ + CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ + tar xfz curl-${CURL_VERSION}.tar.gz && \ + cd curl-${CURL_VERSION} && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ + make -j8 && make install && \ + rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} + +# Pulsar client C++ +RUN PULSAR_CPP_VERSION=$(cat pulsar-client-cpp-version.txt | xargs) && \ + curl -O -L https://archive.apache.org/dist/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \ + tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \ + cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \ + cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \ + make -j8 && \ + make install && \ + rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz diff --git a/pkg/manylinux_musl/Dockerfile b/pkg/manylinux_musl/Dockerfile index f402bec..796d355 100644 --- a/pkg/manylinux_musl/Dockerfile +++ b/pkg/manylinux_musl/Dockerfile @@ -38,6 +38,7 @@ RUN pip install pyyaml RUN apk add cmake ADD .build/dependencies.yaml / +ADD .build/pulsar-client-cpp-version.txt / ADD .build/dep-version.py /usr/local/bin # Download and compile boost @@ -51,3 +52,70 @@ RUN BOOST_VERSION=$(dep-version.py boost) && \ rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} +# Download and compile protobuf +RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + cd protobuf-${PROTOBUF_VERSION}/ && \ + CXXFLAGS=-fPIC ./configure && \ + make -j8 && make install && \ + rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} + +# ZLib +RUN ZLIB_VERSION=$(dep-version.py zlib) && \ + curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ + tar xfz v${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ + CFLAGS="-fPIC -O3" ./configure && \ + make -j8 && make install && \ + rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} + +# Zstandard +RUN ZSTD_VERSION=$(dep-version.py zstd) && \ + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ + tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ + cd zstd-${ZSTD_VERSION} && \ + CFLAGS="-fPIC -O3" make -j8 && \ + make install && \ + rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz + +# Snappy +RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ + curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ + tar xfz ${SNAPPY_VERSION}.tar.gz && \ + cd snappy-${SNAPPY_VERSION} && \ + CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ + make -j8 && make install && \ + rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz + +RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ + OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ + ./config -fPIC --prefix=/usr/local/ssl/ && \ + make -j8 && make install && \ + rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} + +ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ + +# LibCurl +RUN CURL_VERSION=$(dep-version.py curl) && \ + CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ + tar xfz curl-${CURL_VERSION}.tar.gz && \ + cd curl-${CURL_VERSION} && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ + make -j8 && make install && \ + rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} + +# Pulsar client C++ +RUN PULSAR_CPP_VERSION=$(cat pulsar-client-cpp-version.txt | xargs) && \ + curl -O -L https://archive.apache.org/dist/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \ + tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \ + cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \ + cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \ + make -j8 && \ + make install && \ + rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz