Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Python] Added script to generate Wheel files for MacOS #15024

Merged
merged 1 commit into from
Apr 5, 2022
Merged

[Python] Added script to generate Wheel files for MacOS #15024

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions pulsar-client-cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#

cmake_minimum_required(VERSION 3.4)

project (pulsar-cpp)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")

Expand Down Expand Up @@ -117,13 +118,13 @@ endif(NOT LOG_CATEGORY_NAME)

add_definitions(-DLOG_CATEGORY_NAME=${LOG_CATEGORY_NAME} -DBUILDING_PULSAR -DBOOST_ALL_NO_LIB -DBOOST_ALLOW_DEPRECATED_HEADERS)

set(OPENSSL_ROOT_DIR /usr/lib64/)
set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/)

### This part is to find and keep SSL dynamic libs in RECORD_OPENSSL_SSL_LIBRARY and RECORD_OPENSSL_CRYPTO_LIBRARY
### After find the libs, will unset related cache, and will not affact another same call to find_package.
### After find the libs, will unset related cache, and will not affect another same call to find_package.
if (APPLE)
set(OPENSSL_INCLUDE_DIR /usr/local/opt/openssl/include/ /opt/homebrew/opt/openssl/include)
set(OPENSSL_ROOT_DIR /usr/local/opt/openssl/ /opt/homebrew/opt/openssl)
set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl)
endif ()

set(OPENSSL_USE_STATIC_LIBS FALSE)
Expand All @@ -142,11 +143,15 @@ unset(OPENSSL_VERSION CACHE)

if (LINK_STATIC)
find_library(ZLIB_LIBRARIES REQUIRED NAMES libz.a z zlib)
message(STATUS "ZLIB_LIBRARIES: ${ZLIB_LIBRARIES}")
find_library(Protobuf_LIBRARIES NAMES libprotobuf.a libprotobuf)
message(STATUS "Protobuf: ${Protobuf_LIBRARIES}")
find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
find_library(LIB_ZSTD NAMES libzstd.a)
message(STATUS "ZStd: ${LIB_ZSTD}")
find_library(LIB_SNAPPY NAMES libsnappy.a)
message(STATUS "Protobuf_LIBRARIES: ${Protobuf_LIBRARIES}")
message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}")
set(COMMON_LIBS ${Protobuf_LIBRARIES} ${COMMON_LIBS})

if (USE_LOG4CXX)
Expand Down Expand Up @@ -271,7 +276,7 @@ if (BUILD_PYTHON_WRAPPER)
list(GET PYTHONLIBS_VERSION_NO_LIST 1 PYTHONLIBS_VERSION_MINOR)
set(BOOST_PYTHON_NAME_POSTFIX ${PYTHONLIBS_VERSION_MAJOR}${PYTHONLIBS_VERSION_MINOR})
# For python3 the lib name is boost_python3
set(BOOST_PYTHON_NAME_LIST python36;python37;python38;python39;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX})
set(BOOST_PYTHON_NAME_LIST python36;python37;python38;python39;python310;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX})
else ()
# Regular boost_python
set(BOOST_PYTHON_NAME_LIST python;python-mt;python-py27;python27-mt;python27)
Expand All @@ -289,6 +294,7 @@ if (BUILD_PYTHON_WRAPPER)
MESSAGE(FATAL_ERROR "Could not find Boost Python library")
endif ()

MESSAGE(STATUS "BOOST_PYTHON_NAME_FOUND: " ${BOOST_PYTHON_NAME_FOUND})
find_package(Boost REQUIRED COMPONENTS ${BOOST_PYTHON_NAME_FOUND})
endif (BUILD_PYTHON_WRAPPER)

Expand Down
255 changes: 255 additions & 0 deletions pulsar-client-cpp/python/build-mac-wheels.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

set -e

PYTHON_VERSIONS=(
'3.8 3.8.13'
'3.9 3.9.10'
'3.10 3.10.2'
)

export MACOSX_DEPLOYMENT_TARGET=11.0
MACOSX_DEPLOYMENT_TARGET_MAJOR=${MACOSX_DEPLOYMENT_TARGET%%.*}

ZLIB_VERSION=1.2.12
OPENSSL_VERSION=1_1_1n
BOOST_VERSION=1.78.0
PROTOBUF_VERSION=3.20.0
ZSTD_VERSION=1.5.2
SNAPPY_VERSION=1.1.3
CURL_VERSION=7.61.0

ROOT_DIR=$(git rev-parse --show-toplevel)
cd "${ROOT_DIR}/pulsar-client-cpp"


# Compile and cache dependencies
CACHE_DIR=~/.pulsar-mac-wheels-cache
mkdir -p $CACHE_DIR

PREFIX=$CACHE_DIR/install

cd $CACHE_DIR

###############################################################################
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_VERSION_LONG=${PY[1]}

if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then
echo "Building Python $PYTHON_VERSION_LONG"
curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz
tar xfz Python-${PYTHON_VERSION_LONG}.tgz

PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION
pushd Python-${PYTHON_VERSION_LONG}
CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
./configure --prefix=$PY_PREFIX --enable-shared
make -j16
make install

curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl
$PY_PREFIX/bin/pip3 install wheel-*.whl

touch .done
popd
else
echo "Using cached Python $PYTHON_VERSION_LONG"
fi
done

###############################################################################
if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then
echo "Building ZLib"
curl -O -L https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz
tar xvfz zlib-$ZLIB_VERSION.tar.gz
pushd zlib-$ZLIB_VERSION
CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
make -j16
make install
touch .done
popd
else
echo "Using cached ZLib"
fi

###############################################################################
if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION}/.done ]; then
echo "Building OpenSSL"
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.tar.gz
tar xvfz OpenSSL_${OPENSSL_VERSION}.tar.gz
pushd openssl-OpenSSL_${OPENSSL_VERSION}
./Configure -fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} --prefix=$PREFIX no-shared darwin64-arm64-cc
make -j8
make install
touch .done
popd
else
echo "Using cached OpenSSL"
fi

###############################################################################
BOOST_VERSION_=${BOOST_VERSION//./_}
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_VERSION_LONG=${PY[1]}

DIR=boost-src-${BOOST_VERSION}-python-${PYTHON_VERSION}
if [ ! -f $DIR/.done ]; then
echo "Building Boost for Py $PYTHON_VERSION"
curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz
tar xfz boost_${BOOST_VERSION_}.tar.gz
mv boost_${BOOST_VERSION_} $DIR

PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION

pushd $DIR
cat <<EOF > user-config.jam
using python : $PYTHON_VERSION
: python3
: ${PY_PREFIX}/include/python${PYTHON_VERSION}
: ${PY_PREFIX}/lib
;
EOF
./bootstrap.sh --with-libraries=python --with-python=python3 --with-python-root=$PY_PREFIX \
--prefix=$CACHE_DIR/boost-py-$PYTHON_VERSION
./b2 address-model=64 cxxflags="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" link=static threading=multi \
--user-config=./user-config.jam \
variant=release python=${PYTHON_VERSION} \
-j16 \
install
touch .done
popd
else
echo "Using cached Boost for Py $PYTHON_VERSION"
fi

done



###############################################################################
if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then
echo "Building Protobuf"
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
tar xvfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
pushd protobuf-${PROTOBUF_VERSION}
CXXFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
make -j16
make install
touch .done
popd
else
echo "Using cached Protobuf"
fi

###############################################################################
if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then
echo "Building ZStd"
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
tar xvfz zstd-${ZSTD_VERSION}.tar.gz
pushd zstd-${ZSTD_VERSION}
CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX make -j16 install
touch .done
popd
else
echo "Using cached ZStd"
fi

###############################################################################
if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then
echo "Building Snappy"
curl -O -L https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz
tar xvfz snappy-${SNAPPY_VERSION}.tar.gz
pushd snappy-${SNAPPY_VERSION}
CXXFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
make -j16
make install
touch .done
popd
else
echo "Using cached Snappy"
fi

###############################################################################
if [ ! -f curl-${CURL_VERSION}/.done ]; then
echo "Building LibCurl"
CURL_VERSION_=${CURL_VERSION//./_}
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz
tar xfz curl-${CURL_VERSION}.tar.gz
pushd curl-${CURL_VERSION}
CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --with-ssl=$PREFIX \
--without-nghttp2 --without-libidn2 --prefix=$PREFIX
make -j16 install
touch .done
popd
else
echo "Using cached LibCurl"
fi

###############################################################################
###############################################################################
###############################################################################
###############################################################################

for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_VERSION_LONG=${PY[1]}
echo '----------------------------------------------------------------------------'
echo '----------------------------------------------------------------------------'
echo '----------------------------------------------------------------------------'
echo "Build wheel for Python $PYTHON_VERSION"

cd "${ROOT_DIR}/pulsar-client-cpp"

find . -name CMakeCache.txt | xargs -r rm
find . -name CMakeFiles | xargs -r rm -rf

PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION
PY_EXE=$PY_PREFIX/bin/python3

cmake . \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \
-DCMAKE_OSX_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX${MACOSX_DEPLOYMENT_TARGET_MAJOR}.sdk \
-DCMAKE_INSTALL_PREFIX=$PREFIX \
-DCMAKE_INSTALL_LIBDIR=$PREFIX/lib \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH=$PREFIX \
-DCMAKE_CXX_FLAGS=-I$PREFIX/include \
-DCMAKE_FIND_FRAMEWORK=$PREFIX \
-DBoost_INCLUDE_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/include \
-DBoost_LIBRARY_DIRS=$CACHE_DIR/boost-py-$PYTHON_VERSION/lib \
-DPYTHON_INCLUDE_DIR=$PY_PREFIX/include/python$PYTHON_VERSION \
-DPYTHON_LIBRARY=$PY_PREFIX/lib/libpython${PYTHON_VERSION}.dylib \
-DLINK_STATIC=ON \
-DBUILD_TESTS=OFF \
-DBUILD_WIRESHARK=OFF \
-DPROTOC_PATH=$PREFIX/bin/protoc

make clean
make _pulsar -j16

cd python
$PY_EXE setup.py bdist_wheel
done