From 51f92ac042c9bf6643cf509dfb127e0a5311eac9 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Tue, 23 May 2023 22:54:51 +0800 Subject: [PATCH] Support to build wheel package on Linux aarch64 platform (#2723) - Add support for building and installing GraphScope on the Linux aarch64 platform - Fixes some issues related to the learning engine and the `gs` script - Updates the documentation, Makefile, and the `gs` script to handle the aarch64 platform and its dependencies. --- Makefile | 13 ++- docs/learning_engine/getting_started.md | 17 ++- gs | 140 +++++++++--------------- k8s/internal/Makefile | 38 ++++++- python/graphscope/__init__.py | 7 +- python/graphscope/learning/__init__.py | 35 +++++- 6 files changed, 148 insertions(+), 102 deletions(-) diff --git a/Makefile b/Makefile index 897ce87a360c..47e21cdff7b1 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,9 @@ ifeq ($(UNAME),Darwin) SUFFIX := dylib endif +# x86_64 or aarch64 +ARCH := $(shell uname -m) + VERSION := $(shell cat $(WORKING_DIR)/VERSION) ## Common @@ -70,8 +73,14 @@ client: learning cd $(CLIENT_DIR) && \ python3 -m pip install -r requirements.txt -r requirements-dev.txt --user && \ python3 setup.py build_ext --inplace --user - python3 -m pip install --user --editable $(CLIENT_DIR) - rm -rf $(CLIENT_DIR)/*.egg-info + if [[ "${ARCH}" == "aarch64" ]]; then \ + python3 setup.py bdist_wheel; \ + python3 -m pip install --user dist/*.whl; \ + rm -fr $(CLIENT_DIR)/build; \ + else \ + python3 -m pip install --user --editable $(CLIENT_DIR); \ + rm -rf $(CLIENT_DIR)/*.egg-info; \ + fi coordinator: client cd $(COORDINATOR_DIR) && \ diff --git a/docs/learning_engine/getting_started.md b/docs/learning_engine/getting_started.md index c593bd55efab..0aa2edc9fbb8 100644 --- a/docs/learning_engine/getting_started.md +++ b/docs/learning_engine/getting_started.md @@ -26,6 +26,21 @@ you also need to install the tensorflow. python3 -m pip install tensorflow==2.8.0 ``` +Only `v2.11.0` supported under the linux aarch64 platform: + +```python +>>> import platform +>>> platform.system() +'Linux' +>>> platform.processor() +'aarch64' +``` + +```bash +# Install the fixed 'v2.11.0' verion of tensorflow under the linux aarch64 platform +python3 -m pip install tensorflow==2.11.0 +``` + ## Running GraphScope Learning Engine on Local The `graphscope` package includes everything you need to train GNN models @@ -126,4 +141,4 @@ GNN model on your local machine. Next, you may want to learn more about the foll Next, you may want to learn more about the following topics: - [Design of the learning engine of GraphScope and its technical details.](learning_engine/design_of_gle) -- [A set of examples with advanced usage, including deploying GLE in a K8s cluster.](learning_engine/guide_and_exmaples) \ No newline at end of file +- [A set of examples with advanced usage, including deploying GLE in a K8s cluster.](learning_engine/guide_and_exmaples) diff --git a/gs b/gs index 7c12e7496aa0..13513d2963d7 100755 --- a/gs +++ b/gs @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# This script was generated by bashly 0.9.4 (https://bashly.dannyb.co) +# This script was generated by bashly 1.0.2 (https://bashly.dannyb.co) # Modifying it manually is not recommended # :wrapper.bash3_bouncer @@ -96,11 +96,6 @@ gs_make_usage() { if [[ -n $long_usage ]]; then printf "%s\n" "Options:" - # :command.usage_fixed_flags - printf " %s\n" "--help, -h" - printf " Show this help\n" - echo - # :command.usage_flags # :flag.usage printf " %s\n" "--install-prefix PREFIX" @@ -108,6 +103,11 @@ gs_make_usage() { printf " Default: /opt/graphscope\n" echo + # :command.usage_fixed_flags + printf " %s\n" "--help, -h" + printf " Show this help\n" + echo + # :command.usage_args printf "%s\n" "Arguments:" @@ -152,11 +152,6 @@ gs_make_image_usage() { if [[ -n $long_usage ]]; then printf "%s\n" "Options:" - # :command.usage_fixed_flags - printf " %s\n" "--help, -h" - printf " Show this help\n" - echo - # :command.usage_flags # :flag.usage printf " %s\n" "--registry REGISTRY" @@ -175,6 +170,11 @@ gs_make_image_usage() { printf " Whether to use CN located mirrors to speed up download. [Not implemented]\n" echo + # :command.usage_fixed_flags + printf " %s\n" "--help, -h" + printf " Show this help\n" + echo + # :command.usage_args printf "%s\n" "Arguments:" @@ -217,17 +217,17 @@ gs_dev_usage() { if [[ -n $long_usage ]]; then printf "%s\n" "Options:" - # :command.usage_fixed_flags - printf " %s\n" "--help, -h" - printf " Show this help\n" - echo - # :command.usage_flags # :flag.usage printf " %s\n" "--local, -l LOCAL" printf " Local path to the source code of GraphScope.\n" echo + # :command.usage_fixed_flags + printf " %s\n" "--help, -h" + printf " Show this help\n" + echo + # :command.usage_examples printf "%s\n" "Examples:" printf " gs dev\n" @@ -261,11 +261,6 @@ gs_test_usage() { if [[ -n $long_usage ]]; then printf "%s\n" "Options:" - # :command.usage_fixed_flags - printf " %s\n" "--help, -h" - printf " Show this help\n" - echo - # :command.usage_flags # :flag.usage printf " %s\n" "--local" @@ -288,6 +283,11 @@ gs_test_usage() { printf " Default: /tmp/gstest\n" echo + # :command.usage_fixed_flags + printf " %s\n" "--help, -h" + printf " Show this help\n" + echo + # :command.usage_args printf "%s\n" "Arguments:" @@ -324,11 +324,6 @@ gs_install_deps_usage() { if [[ -n $long_usage ]]; then printf "%s\n" "Options:" - # :command.usage_fixed_flags - printf " %s\n" "--help, -h" - printf " Show this help\n" - echo - # :command.usage_flags # :flag.usage printf " %s\n" "--cn" @@ -368,6 +363,11 @@ gs_install_deps_usage() { printf " Do not install v6d, for build base docker images, could only be used with\n '--for-analytical'\n" echo + # :command.usage_fixed_flags + printf " %s\n" "--help, -h" + printf " Show this help\n" + echo + # :command.usage_args printf "%s\n" "Arguments:" @@ -454,8 +454,8 @@ normalize_input() { } # :command.inspect_args inspect_args() { - readarray -t sorted_keys < <(printf '%s\n' "${!args[@]}" | sort) if ((${#args[@]})); then + readarray -t sorted_keys < <(printf '%s\n' "${!args[@]}" | sort) echo args: for k in "${sorted_keys[@]}"; do echo "- \${args[$k]} = ${args[$k]}"; done else @@ -470,6 +470,14 @@ inspect_args() { echo "- \${other_args[$i]} = ${other_args[$i]}" done fi + + if ((${#deps[@]})); then + readarray -t sorted_keys < <(printf '%s\n' "${!deps[@]}" | sort) + echo + echo deps: + for k in "${sorted_keys[@]}"; do echo "- \${deps[$k]} = ${deps[$k]}"; done + fi + } # :command.user_lib @@ -1019,14 +1027,7 @@ install_vineyard() { make -j"${jobs}" make install strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard* - python3 setup.py bdist_wheel - # This is output fixed wheels to wheelhouse/ - python3 -m auditwheel repair dist/* - rm -rf dist/* - python3 setup_bdist.py bdist_wheel - python3 setup_io.py bdist_wheel - mv dist/*.whl wheelhouse/ - pip3 install --no-cache wheelhouse/* --user + pip3 install "vineyard==${v6d_version}" cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/ popd || exit popd || exit @@ -2056,7 +2057,9 @@ gs_make_image_parse_requirements() { done # :command.dependencies_filter - if ! command -v docker >/dev/null 2>&1; then + if command -v docker >/dev/null 2>&1; then + deps['docker']="$(command -v docker | head -n1)" + else printf "missing dependency: docker\n" >&2 exit 1 fi @@ -2172,7 +2175,9 @@ gs_dev_parse_requirements() { done # :command.dependencies_filter - if ! command -v docker >/dev/null 2>&1; then + if command -v docker >/dev/null 2>&1; then + deps['docker']="$(command -v docker | head -n1)" + else printf "missing dependency: docker\n" >&2 exit 1 fi @@ -2558,66 +2563,19 @@ initialize() { # :command.run run() { declare -A args=() + declare -A deps=() declare -a other_args=() declare -a input=() normalize_input "$@" parse_requirements "${input[@]}" case "$action" in - "make") - if [[ ${args['--help']:-} ]]; then - long_usage=yes - gs_make_usage - else - gs_make_command - fi - ;; - - "make-image") - if [[ ${args['--help']:-} ]]; then - long_usage=yes - gs_make_image_usage - else - gs_make_image_command - fi - ;; - - "dev") - if [[ ${args['--help']:-} ]]; then - long_usage=yes - gs_dev_usage - else - gs_dev_command - fi - ;; - - "test") - if [[ ${args['--help']:-} ]]; then - long_usage=yes - gs_test_usage - else - gs_test_command - fi - ;; - - "install-deps") - if [[ ${args['--help']:-} ]]; then - long_usage=yes - gs_install_deps_usage - else - gs_install_deps_command - fi - ;; - - "format") - if [[ ${args['--help']:-} ]]; then - long_usage=yes - gs_format_usage - else - gs_format_command - fi - ;; - + "make") gs_make_command ;; + "make-image") gs_make_image_command ;; + "dev") gs_dev_command ;; + "test") gs_test_command ;; + "install-deps") gs_install_deps_command ;; + "format") gs_format_command ;; esac } diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index db0edfb37070..ad172a017ab7 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -102,6 +102,9 @@ graphscope-darwin-py3: graphscope-manylinux2014-py3-nodocker: cd $(WORKING_DIR)/../.. && \ + if [[ "${ARCH}" == "aarch64" ]]; then python3 -m pip install grpcio==1.49.1 --no-binary grpcio; \ + export AUDITWHEEL_PLAT=manylinux2014_${ARCH}; \ + python3 -m pip install grpcio-tools==1.49.1 --no-binary grpcio-tools; fi && \ for _ in {1..5}; do if make INSTALL_PREFIX=${INSTALL_PREFIX}; then break; fi; done && \ for _ in {1..5}; do if $(MAKE_INSTALL_COMMAND); then break; fi; done && \ if [ ${INSTALL_PREFIX} != ${GRAPHSCOPE_HOME} ]; then sudo cp -rsn ${INSTALL_PREFIX}/* ${GRAPHSCOPE_HOME}/; fi && \ @@ -150,24 +153,49 @@ graphscope-client-manylinux2014-py3-nodocker: cmake -DKNN=OFF -DWITH_VINEYARD=ON -DTESTING=OFF .. && \ make graphlearn_shared -j`nproc` && \ export LD_LIBRARY_PATH=$(WORKING_DIR)/../../learning_engine/graph-learn/graphlearn/built/lib:$$LD_LIBRARY_PATH && \ + if [[ "${ARCH}" == "aarch64" ]]; then export AUDITWHEEL_PLAT=manylinux2014_${ARCH}; fi && \ for py in cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311; do \ cd $(WORKING_DIR)/../../python; \ export PATH=/opt/python/$$py/bin:$$PATH; \ python3 -m pip install -U pip; \ if [[ "$$py" == "cp311-cp311" ]]; then \ - python3 -m pip install "numpy==1.23.2" "pandas" "grpcio" "grpcio-tools" wheel "auditwheel==5.0.0"; \ + if [[ "${ARCH}" == "aarch64" ]]; then \ + python3 -m pip install grpcio==1.49.1 --no-binary grpcio; \ + python3 -m pip install grpcio-tools==1.49.1 --no-binary grpcio-tools; \ + python3 -m pip install "numpy==1.23.2" "pandas" wheel "auditwheel==5.0.0"; \ + else \ + python3 -m pip install "numpy==1.23.2" "pandas" "grpcio" "grpcio-tools" wheel "auditwheel==5.0.0"; \ + fi; \ elif [[ "$$py" == "cp310-cp310" ]]; then \ - python3 -m pip install "numpy==1.21.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ + if [[ "${ARCH}" == "aarch64" ]]; then \ + python3 -m pip install grpcio==1.49.1 --no-binary grpcio; \ + python3 -m pip install grpcio-tools==1.49.1 --no-binary grpcio-tools; \ + python3 -m pip install "numpy==1.21.2" "pandas" wheel "auditwheel==5.0.0"; \ + else \ + python3 -m pip install "numpy==1.21.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ + fi; \ elif [[ "$$py" == "cp39-cp39" ]]; then \ - python3 -m pip install "numpy==1.19.3" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ + if [[ "${ARCH}" == "aarch64" ]]; then \ + python3 -m pip install grpcio==1.49.1 --no-binary grpcio; \ + python3 -m pip install grpcio-tools==1.49.1 --no-binary grpcio-tools; \ + python3 -m pip install "numpy==1.19.3" "pandas" wheel "auditwheel==5.0.0"; \ + else \ + python3 -m pip install "numpy==1.19.3" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ + fi; \ else \ - python3 -m pip install "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ + if [[ "${ARCH}" == "aarch64" ]]; then \ + python3 -m pip install grpcio==1.49.1 --no-binary grpcio; \ + python3 -m pip install grpcio-tools==1.49.1 --no-binary grpcio-tools; \ + python3 -m pip install "numpy" "pandas" wheel "auditwheel==5.0.0"; \ + else \ + python3 -m pip install "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ + fi; \ fi; \ sudo rm -rf build; \ sudo rm -rf dist/*.whl; \ python3 setup.py bdist_wheel; \ cd dist; \ - auditwheel repair ./*.whl; \ + auditwheel repair ./*.whl --plat=manylinux2014_${ARCH}; \ done graphscope-client-darwin-py3: diff --git a/python/graphscope/__init__.py b/python/graphscope/__init__.py index f491a4edc15c..13a1ce379a98 100644 --- a/python/graphscope/__init__.py +++ b/python/graphscope/__init__.py @@ -17,10 +17,13 @@ # import os +import platform import sys -# Tensorflow with Python 3.7 requires lower version of protobuf -if sys.version_info.major == 3 and sys.version_info.minor == 7: +# Tensorflow with Python 3.7 and ARM platform requires lower version of protobuf +if (sys.version_info.major == 3 and sys.version_info.minor == 7) or ( + platform.system() == "Linux" and platform.processor() == "aarch64" +): os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" # The gremlinpython has a async event loop, which may conflicts with diff --git a/python/graphscope/learning/__init__.py b/python/graphscope/learning/__init__.py index 87b0b14c45a4..c3afdafae417 100644 --- a/python/graphscope/learning/__init__.py +++ b/python/graphscope/learning/__init__.py @@ -20,6 +20,39 @@ import platform import sys + +def _force_preload_libgomp(): + """Load libgomp before importing tensorflow. See also: + - https://github.com/opencv/opencv/issues/14884 + - https://github.com/pytorch/pytorch/issues/2575 + - https://github.com/dmlc/xgboost/issues/7110#issuecomment-880841484 + """ + if platform.system() != "Linux" or platform.processor() != "aarch64": + return + + import ctypes + import glob + + pkgs_directory = os.path.join( + os.path.abspath(os.path.dirname(__file__)), "..", ".." + ) + for lib_directory in [ + os.path.join(pkgs_directory, "graphscope_client.libs"), + os.path.join(pkgs_directory, "tensorflow_cpu_aws.libs"), + ]: + for libfile in glob.glob(lib_directory + "/libgomp*.so*"): + import sys + + try: + ctypes.cdll.LoadLibrary(libfile) + except: # noqa: E722, pylint: disable=bare-except + pass + + +_force_preload_libgomp() +del _force_preload_libgomp + + try: sys.path.insert(0, os.path.dirname(__file__)) @@ -60,7 +93,7 @@ def reset_default_tf_graph(): if tf is not None: try: tf.reset_default_graph() - except: # noqa: E722, pylint: disable=bare-except + except: # noqa pass ctx = {"GRPC_VERBOSITY": "NONE"}