diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 945b8e4f72..fd6b8c1879 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,7 +66,7 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - wheel-build-cuml: + wheel-build-libcuml: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: @@ -74,13 +74,36 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel.sh + script: ci/build_wheel_libcuml.sh # Note that this approach to cloning repos obviates any modification to # the CMake variables in get_cumlprims_mg.cmake since CMake will just use # the clone as is. extra-repo: rapidsai/cumlprims_mg extra-repo-sha: branch-25.02 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + wheel-publish-libcuml: + needs: wheel-build-libcuml + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcuml + package-type: cpp + wheel-build-cuml: + needs: wheel-build-libcuml + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cuml.sh wheel-publish-cuml: needs: wheel-build-cuml secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 95d648f8e6..37b176d635 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,7 @@ jobs: - conda-notebook-tests - docs-build - telemetry-setup + - wheel-build-libcuml - wheel-build-cuml - wheel-tests-cuml - devcontainer @@ -166,16 +167,28 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cuml: + wheel-build-libcuml: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request - script: ci/build_wheel.sh + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcuml.sh extra-repo: rapidsai/cumlprims_mg extra-repo-sha: branch-25.02 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + wheel-build-cuml: + needs: [checks, wheel-build-libcuml] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: pull-request + script: ci/build_wheel_cuml.sh wheel-tests-cuml: needs: [wheel-build-cuml, changed-files] secrets: inherit diff --git a/build.sh b/build.sh index 9eb36f103c..6f1b6b9f83 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2019-2024, NVIDIA CORPORATION. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. # cuml build script @@ -288,11 +288,6 @@ if (! hasArg --configure-only) && (completeBuild || hasArg cuml || hasArg pydocs # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') - # Append `-DFIND_CUML_CPP=ON` to CUML_EXTRA_CMAKE_ARGS unless a user specified the option. - if [[ "${SKBUILD_EXTRA_CMAKE_ARGS}" != *"DFIND_CUML_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DFIND_CUML_CPP=ON" - fi - SKBUILD_CMAKE_ARGS="-DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL};${SKBUILD_EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python/cuml diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index fa066c03c6..7c35f35be5 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -9,6 +9,9 @@ source rapids-configure-sccache source rapids-date-string +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + export CMAKE_GENERATOR=Ninja rapids-print-env diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 05f1f24ee5..51d46ef738 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -1,10 +1,13 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + RAPIDS_VERSION="$(rapids-version)" export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" diff --git a/ci/build_python.sh b/ci/build_python.sh index e8d2b3ed39..d0b5724f89 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -9,6 +9,9 @@ source rapids-configure-sccache source rapids-date-string +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + export CMAKE_GENERATOR=Ninja rapids-print-env diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 104895daf7..3c840d9849 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,55 +1,26 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail -package_dir="python/cuml" +package_name=$1 +package_dir=$2 source rapids-configure-sccache source rapids-date-string -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - rapids-generate-version > ./VERSION -cd ${package_dir} - -case "${RAPIDS_CUDA_VERSION}" in - 12.*) - EXCLUDE_ARGS=( - --exclude "libcuvs.so" - --exclude "libcublas.so.12" - --exclude "libcublasLt.so.12" - --exclude "libcufft.so.11" - --exclude "libcurand.so.10" - --exclude "libcusolver.so.11" - --exclude "libcusparse.so.12" - --exclude "libnvJitLink.so.12" - ) - EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" - ;; - 11.*) - EXCLUDE_ARGS=( - --exclude "libcuvs.so" - ) - EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" - ;; -esac +cd "${package_dir}" sccache --zero-stats -SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/;-DUSE_CUVS_WHEEL=ON${EXTRA_CMAKE_ARGS}" \ - python -m pip wheel . \ +rapids-logger "Building '${package_name}' wheel" +python -m pip wheel \ -w dist \ -v \ --no-deps \ - --disable-pip-version-check + --disable-pip-version-check \ + . sccache --show-adv-stats - -mkdir -p final_dist -python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* - -../../ci/validate_wheel.sh final_dist - -RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist diff --git a/ci/build_wheel_cuml.sh b/ci/build_wheel_cuml.sh new file mode 100755 index 0000000000..d35ea9893c --- /dev/null +++ b/ci/build_wheel_cuml.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) 2023-2025, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="cuml" +package_dir="python/cuml" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ./ci/use_wheels_from_prs.sh + +# Download the libcuml wheel built in the previous step and make it +# available for pip to find. +RAPIDS_PY_WHEEL_NAME="libcuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcuml_dist +echo "libcuml-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcuml_dist/libcuml_*.whl)" >> /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" + +EXCLUDE_ARGS=( + --exclude "libcuml++.so" + --exclude "libcumlprims_mg.so" + --exclude "libcuvs.so" + --exclude "libraft.so" + --exclude "libcublas.so.*" + --exclude "libcublasLt.so.*" + --exclude "libcufft.so.*" + --exclude "libcurand.so.*" + --exclude "libcusolver.so.*" + --exclude "libcusparse.so.*" + --exclude "libnvJitLink.so.*" +) + +export SKBUILD_CMAKE_ARGS="-DDISABLE_DEPRECATION_WARNINGS=ON;-DSINGLEGPU=OFF;-DUSE_LIBCUML_WHEEL=ON" +./ci/build_wheel.sh "${package_name}" "${package_dir}" + +mkdir -p ${package_dir}/final_dist +python -m auditwheel repair \ + "${EXCLUDE_ARGS[@]}" \ + -w ${package_dir}/final_dist \ + ${package_dir}/dist/* + +./ci/validate_wheel.sh ${package_dir} final_dist + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${package_dir}/final_dist" diff --git a/ci/build_wheel_libcuml.sh b/ci/build_wheel_libcuml.sh new file mode 100755 index 0000000000..be4c6365d3 --- /dev/null +++ b/ci/build_wheel_libcuml.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright (c) 2025, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libcuml" +package_dir="python/libcuml" + +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ./ci/use_wheels_from_prs.sh + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + +# NOTE: 'libcumlprims_mg.so' is marked as '--exclude' here because auditwheel doesn't detect it, +# but it really is intentionally included in 'libcuml' wheels +EXCLUDE_ARGS=( + --exclude "libcumlprims_mg.so" + --exclude "libcuvs.so" + --exclude "libraft.so" + --exclude "libcublas.so.*" + --exclude "libcublasLt.so.*" + --exclude "libcufft.so.*" + --exclude "libcurand.so.*" + --exclude "libcusolver.so.*" + --exclude "libcusparse.so.*" + --exclude "libnvJitLink.so.*" +) + +export SKBUILD_CMAKE_ARGS="-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/" +./ci/build_wheel.sh "${package_name}" "${package_dir}" + +mkdir -p ${package_dir}/final_dist +python -m auditwheel repair \ + "${EXCLUDE_ARGS[@]}" \ + -w ${package_dir}/final_dist \ + ${package_dir}/dist/* + +./ci/validate_wheel.sh ${package_dir} final_dist + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp "${package_dir}/final_dist" diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index ea6d1cdc11..5fc351e15b 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -8,6 +8,9 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index d76a754b8b..92c63860e2 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -1,9 +1,12 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 5f1894356c..3a2c772c51 100644 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -1,10 +1,13 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 76cfe9a86b..7fa04ef5d7 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,12 +3,17 @@ set -euo pipefail +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ./ci/use_wheels_from_prs.sh + mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="libcuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ + ./dist/libcuml*.whl \ "$(echo ./dist/cuml*.whl)[test]" RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh new file mode 100644 index 0000000000..59b7eb3511 --- /dev/null +++ b/ci/use_conda_packages_from_prs.sh @@ -0,0 +1,10 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# TODO(jameslamb): remove this file when https://github.com/rapidsai/cuvs/pull/594 is merged + +CUVS_COMMIT="86405194a1768b72fe4f8fcd7e7894e2a0b135c7" + +CUVS_CPP_CHANNEL=$(rapids-get-pr-conda-artifact cuvs 594 cpp "${CUVS_COMMIT:0:7}") +CUVS_PYTHON_CHANNEL=$(rapids-get-pr-conda-artifact cuvs 594 python "${CUVS_COMMIT:0:7}") + +conda config --system --add channels "${CUVS_CPP_CHANNEL}" +conda config --system --add channels "${CUVS_PYTHON_CHANNEL}" diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh new file mode 100644 index 0000000000..ec85b54d5c --- /dev/null +++ b/ci/use_wheels_from_prs.sh @@ -0,0 +1,19 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# TODO(jameslamb): remove this file when https://github.com/rapidsai/cuvs/pull/594 is merged + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +CUVS_COMMIT="86405194a1768b72fe4f8fcd7e7894e2a0b135c7" +CUVS_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME="cuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cuvs 594 python "${CUVS_COMMIT:0:7}" +) +LIBCUVS_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME="libcuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact cuvs 594 cpp "${CUVS_COMMIT:0:7}" +) + +cat > /tmp/constraints.txt <=0.0.0a0 -- libraft-headers==25.2.*,>=0.0.0a0 +- libraft==25.2.*,>=0.0.0a0 - librmm==25.2.*,>=0.0.0a0 - nbsphinx - ninja diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 5c8439cf50..f83a242a43 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -39,7 +39,7 @@ dependencies: - libcusolver-dev - libcusparse-dev - libcuvs==25.2.*,>=0.0.0a0 -- libraft-headers==25.2.*,>=0.0.0a0 +- libraft==25.2.*,>=0.0.0a0 - librmm==25.2.*,>=0.0.0a0 - nbsphinx - ninja diff --git a/cpp/cmake/thirdparty/get_cuvs.cmake b/cpp/cmake/thirdparty/get_cuvs.cmake index a48b4c6b1e..7a2b5f3571 100644 --- a/cpp/cmake/thirdparty/get_cuvs.cmake +++ b/cpp/cmake/thirdparty/get_cuvs.cmake @@ -54,8 +54,8 @@ function(find_and_configure_cuvs) EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL} OPTIONS "BUILD_TESTS OFF" - "BUILD_BENCH OFF" "BUILD_CAGRA_HNSWLIB OFF" + "BUILD_CUVS_BENCH OFF" "BUILD_MG_ALGOS ${CUVS_BUILD_MG_ALGOS}" ) diff --git a/cpp/cmake/thirdparty/get_treelite.cmake b/cpp/cmake/thirdparty/get_treelite.cmake index a14bacb531..e197b76a11 100644 --- a/cpp/cmake/thirdparty/get_treelite.cmake +++ b/cpp/cmake/thirdparty/get_treelite.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2021-2024, NVIDIA CORPORATION. +# Copyright (c) 2021-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ function(find_and_configure_treelite) rapids_cpm_find(Treelite ${PKG_VERSION} GLOBAL_TARGETS ${TREELITE_LIBS} + BUILD_EXPORT_SET cuml-exports INSTALL_EXPORT_SET cuml-exports CPM_ARGS GIT_REPOSITORY https://github.com/dmlc/treelite.git diff --git a/dependencies.yaml b/dependencies.yaml index 6761744857..48b8da69ad 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -16,7 +16,7 @@ files: - depends_on_dask_cudf - depends_on_libcumlprims - depends_on_libcuvs - - depends_on_libraft_headers + - depends_on_libraft - depends_on_librmm - depends_on_pylibraft - depends_on_raft_dask @@ -110,9 +110,11 @@ files: includes: - common_build - depends_on_cuda_python - - depends_on_cuvs + - depends_on_libcuml - depends_on_libcumlprims - - depends_on_libraft_headers + - depends_on_libcuvs + - depends_on_libraft + - depends_on_librmm - depends_on_pylibraft - depends_on_rmm - py_build_cuml @@ -128,6 +130,7 @@ files: - depends_on_cupy - depends_on_cuvs - depends_on_dask_cudf + - depends_on_libcuml - depends_on_pylibraft - depends_on_raft_dask - depends_on_rmm @@ -140,6 +143,33 @@ files: key: test includes: - test_python + py_build_libcuml: + output: pyproject + pyproject_dir: python/libcuml + extras: + table: build-system + includes: + - rapids_build_backend + py_rapids_build_libcuml: + output: pyproject + pyproject_dir: python/libcuml + extras: + table: tool.rapids-build-backend + key: requires + includes: + - common_build + - depends_on_libcuvs + - depends_on_libraft + - depends_on_librmm + py_run_libcuml: + output: pyproject + pyproject_dir: python/libcuml + extras: + table: project + includes: + - cuda_wheels + - depends_on_libcuvs + - depends_on_libraft channels: - rapidsai - rapidsai-nightly @@ -370,6 +400,11 @@ dependencies: cuda: "11.*" use_cuda_wheels: "true" packages: + - nvidia-cublas-cu11 + - nvidia-cufft-cu11 + - nvidia-curand-cu11 + - nvidia-cusparse-cu11 + - nvidia-cusolver-cu11 # if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels # (e.g. for DLFW and pip devcontainers) - matrix: @@ -583,7 +618,26 @@ dependencies: common: - output_types: conda packages: - - libcuml==25.2.*,>=0.0.0a0 + - &libcuml_unsuffixed libcuml==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libcuml-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libcuml-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*libcuml_unsuffixed]} depends_on_libcumlprims: common: - output_types: conda @@ -593,7 +647,50 @@ dependencies: common: - output_types: conda packages: - - libcuvs==25.2.*,>=0.0.0a0 + - &libcuvs_unsuffixed libcuvs==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libcuvs-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libcuvs-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*libcuvs_unsuffixed]} + depends_on_libraft: + common: + - output_types: conda + packages: + - &libraft_unsuffixed libraft==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libraft-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libraft-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*libraft_unsuffixed]} depends_on_libraft_headers: common: - output_types: conda @@ -603,7 +700,26 @@ dependencies: common: - output_types: conda packages: - - librmm==25.2.*,>=0.0.0a0 + - &librmm_unsuffixed librmm==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*librmm_unsuffixed]} depends_on_pylibraft: common: - output_types: conda diff --git a/python/cuml/CMakeLists.txt b/python/cuml/CMakeLists.txt index 7bda8d9a64..c491420d71 100644 --- a/python/cuml/CMakeLists.txt +++ b/python/cuml/CMakeLists.txt @@ -36,17 +36,12 @@ project( ################################################################################ # - User Options -------------------------------------------------------------- option(CUML_UNIVERSAL "Build all cuML Python components." ON) -option(FIND_CUML_CPP "Search for existing CUML C++ installations before defaulting to local files" OFF) option(SINGLEGPU "Disable all mnmg components and comms libraries" OFF) -option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) -option(USE_CUVS_WHEEL "Use the cuVS wheel" OFF) -set(CUML_RAFT_CLONE_ON_PIN OFF) - +option(USE_LIBCUML_WHEEL "Use libcuml wheel to provide some dependencies" OFF) # todo: use CMAKE_MESSAGE_CONTEXT for prefix for logging. # https://github.com/rapidsai/cuml/issues/4843 message(VERBOSE "CUML_PY: Build only cuML CPU Python components.: ${CUML_CPU}") -message(VERBOSE "CUML_PY: Searching for existing CUML C++ installations before defaulting to local files: ${FIND_CUML_CPP}") message(VERBOSE "CUML_PY: Disabling all mnmg components and comms libraries: ${SINGLEGPU}") set(CUML_ALGORITHMS "ALL" CACHE STRING "Choose which algorithms are built cuML. Can specify individual algorithms or groups in a semicolon-separated list.") @@ -57,83 +52,11 @@ set(CUML_CPP_SRC "../../cpp") ################################################################################ # - Process User Options ------------------------------------------------------ -# If the user requested it, we attempt to find cuml. -if(FIND_CUML_CPP) - # We need to call get_treelite explicitly because we need the correct - # ${TREELITE_LIBS} definition for RF - include(rapids-cpm) - include(rapids-export) - rapids_cpm_init() - find_package(cuml ${CUML_VERSION} REQUIRED) - include(${CUML_CPP_SRC}/cmake/thirdparty/get_treelite.cmake) -else() - set(cuml_FOUND OFF) -endif() - -include(rapids-cython-core) - -set(CUML_PYTHON_TREELITE_TARGET treelite::treelite) - -if(NOT CUML_CPU) - if(NOT cuml_FOUND) - find_package(CUDAToolkit REQUIRED) - - set(BUILD_CUML_TESTS OFF) - set(BUILD_PRIMS_TESTS OFF) - set(BUILD_CUML_C_LIBRARY OFF) - set(BUILD_CUML_EXAMPLES OFF) - set(BUILD_CUML_BENCH OFF) - set(CUML_EXPORT_TREELITE_LINKAGE ON) - set(CUML_PYTHON_TREELITE_TARGET treelite::treelite_static) - - # Statically link dependencies if building wheels - set(CUDA_STATIC_RUNTIME ON) - set(CUML_USE_CUVS_STATIC ON) - set(CUML_USE_TREELITE_STATIC ON) - set(CUML_USE_CUMLPRIMS_MG_STATIC ON) - # Link to the CUDA wheels with shared libraries for CUDA 12+ - if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) - set(CUDA_STATIC_MATH_LIBRARIES OFF) - else() - if(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") - endif() - set(CUDA_STATIC_MATH_LIBRARIES ON) - endif() - # Don't install the static libs into wheels - set(CUML_EXCLUDE_RAFT_FROM_ALL ON) - set(RAFT_EXCLUDE_FAISS_FROM_ALL ON) - set(CUML_EXCLUDE_TREELITE_FROM_ALL ON) - set(CUML_EXCLUDE_CUMLPRIMS_MG_FROM_ALL ON) - - add_subdirectory(${CUML_CPP_SRC} cuml-cpp EXCLUDE_FROM_ALL) - - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set(rpaths - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/cufft/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" - ) - set_property(TARGET ${CUML_CPP_TARGET} PROPERTY INSTALL_RPATH ${rpaths} APPEND) - endif() - - if(USE_CUVS_WHEEL) - set(rpaths "$ORIGIN/../cuvs") - set_property(TARGET ${CUML_CPP_TARGET} PROPERTY INSTALL_RPATH ${rpaths} APPEND) - endif() - - set(cython_lib_dir cuml) - install(TARGETS ${CUML_CPP_TARGET} DESTINATION ${cython_lib_dir}) - endif() -endif() - if(CUML_CPU) set(CUML_UNIVERSAL OFF) set(SINGLEGPU ON) + # only a subset of algorithms are supported in CPU-only cuML set(CUML_ALGORITHMS "linearregression") list(APPEND CUML_ALGORITHMS "pca") list(APPEND CUML_ALGORITHMS "tsvd") @@ -152,22 +75,49 @@ if(CUML_CPU) list(APPEND CYTHON_FLAGS "--compile-time-env GPUBUILD=0") + +# cuml-cpu does not need libcuml++.so else() + + include(rapids-cpm) + include(rapids-export) + rapids_cpm_init() + + # --- treelite --- # + # Need to call get_treelite explicitly because we need the correct + # ${TREELITE_LIBS} definition for RF. + # + # And because cuml Cython code needs the headers to satisfy calls like + # 'cdef extern from "treelite/c_api.h"' + + # wheel builds use a static treelite, because the 'libtreelite.so' in 'treelite' wheels + # isn't intended for dynamic linking by third-party projects (e.g. hides its symbols) + if(USE_LIBCUML_WHEEL) + set(CUML_PYTHON_TREELITE_TARGET treelite::treelite_static) + set(CUML_USE_TREELITE_STATIC ON) + else() + set(CUML_PYTHON_TREELITE_TARGET treelite::treelite) + set(CUML_USE_TREELITE_STATIC OFF) + endif() + + set(CUML_EXCLUDE_TREELITE_FROM_ALL ON) + + include(${CUML_CPP_SRC}/cmake/thirdparty/get_treelite.cmake) + + # --- libcuml --- # + find_package(cuml "${RAPIDS_VERSION}" REQUIRED) + set(cuml_sg_libraries cuml::${CUML_CPP_TARGET}) set(cuml_mg_libraries cuml::${CUML_CPP_TARGET}) + if(NOT SINGLEGPU) + list(APPEND cuml_mg_libraries cumlprims_mg::cumlprims_mg) + endif() + list(APPEND CYTHON_FLAGS "--compile-time-env GPUBUILD=1") endif() -if(NOT SINGLEGPU) - include("${CUML_CPP_SRC}/cmake/thirdparty/get_cumlprims_mg.cmake") - set(cuml_mg_libraries - cuml::${CUML_CPP_TARGET} - cumlprims_mg::cumlprims_mg - ) -endif() - ################################################################################ # - Build Cython artifacts ----------------------------------------------------- @@ -182,6 +132,7 @@ endif() message(VERBOSE "CUML_PY: Building cuML with algorithms: '${CUML_ALGORITHMS}'.") +include(rapids-cython-core) rapids_cython_init() add_subdirectory(cuml/common) @@ -206,11 +157,3 @@ add_subdirectory(cuml/svm) add_subdirectory(cuml/tsa) add_subdirectory(cuml/experimental/linear_model) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET cuml PATHS "${cython_lib_dir}") -endif() - -if(USE_CUVS_WHEEL) - rapids_cython_add_rpath_entries(TARGET cuml PATHS cuvs) -endif() diff --git a/python/cuml/cuml/__init__.py b/python/cuml/cuml/__init__.py index 62ab93c1b4..a8557c84fc 100644 --- a/python/cuml/cuml/__init__.py +++ b/python/cuml/cuml/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,16 @@ # limitations under the License. # +# If libcuml was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libcuml +except ModuleNotFoundError: + pass +else: + libcuml.load_library() + del libcuml + from cuml.internals.base import Base, UniversalBase from cuml.internals.available_devices import is_cuda_available diff --git a/python/cuml/cuml/cluster/CMakeLists.txt b/python/cuml/cuml/cluster/CMakeLists.txt index 43d15ae6fa..ac012ff510 100644 --- a/python/cuml/cuml/cluster/CMakeLists.txt +++ b/python/cuml/cuml/cluster/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -31,5 +31,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_mg_libraries}" MODULE_PREFIX cluster_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/cluster/hdbscan/CMakeLists.txt b/python/cuml/cuml/cluster/hdbscan/CMakeLists.txt index 2c4b41909d..5a85a97b2d 100644 --- a/python/cuml/cuml/cluster/hdbscan/CMakeLists.txt +++ b/python/cuml/cuml/cluster/hdbscan/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -21,5 +21,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX cluster_hdbscan_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/common/CMakeLists.txt b/python/cuml/cuml/common/CMakeLists.txt index 1492dcd46a..df6cee81cb 100644 --- a/python/cuml/cuml/common/CMakeLists.txt +++ b/python/cuml/cuml/common/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -28,7 +28,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_mg_libraries}" MODULE_PREFIX common_ - ASSOCIATED_TARGETS cuml ) if(${CUML_UNIVERSAL}) diff --git a/python/cuml/cuml/datasets/CMakeLists.txt b/python/cuml/cuml/datasets/CMakeLists.txt index 51d6614600..64c2b483f6 100644 --- a/python/cuml/cuml/datasets/CMakeLists.txt +++ b/python/cuml/cuml/datasets/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -21,5 +21,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX datasets_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/decomposition/CMakeLists.txt b/python/cuml/cuml/decomposition/CMakeLists.txt index 2552c80d74..71f36d57af 100644 --- a/python/cuml/cuml/decomposition/CMakeLists.txt +++ b/python/cuml/cuml/decomposition/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -30,5 +30,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_mg_libraries}" MODULE_PREFIX decomposition_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/ensemble/CMakeLists.txt b/python/cuml/cuml/ensemble/CMakeLists.txt index e3732c1577..38999083e6 100644 --- a/python/cuml/cuml/ensemble/CMakeLists.txt +++ b/python/cuml/cuml/ensemble/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -27,5 +27,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX ensemble_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/experimental/fil/CMakeLists.txt b/python/cuml/cuml/experimental/fil/CMakeLists.txt index f558a47620..d2baab6642 100644 --- a/python/cuml/cuml/experimental/fil/CMakeLists.txt +++ b/python/cuml/cuml/experimental/fil/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -26,5 +26,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX experimental_fil_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/experimental/linear_model/CMakeLists.txt b/python/cuml/cuml/experimental/linear_model/CMakeLists.txt index 6c52c3cd16..eb367d9a56 100644 --- a/python/cuml/cuml/experimental/linear_model/CMakeLists.txt +++ b/python/cuml/cuml/experimental/linear_model/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,5 +22,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX experimental_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/explainer/CMakeLists.txt b/python/cuml/cuml/explainer/CMakeLists.txt index e982fb1264..213bb74a78 100644 --- a/python/cuml/cuml/explainer/CMakeLists.txt +++ b/python/cuml/cuml/explainer/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -28,5 +28,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX explainer_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/fil/CMakeLists.txt b/python/cuml/cuml/fil/CMakeLists.txt index 54e2df2cd4..816e8aa7c8 100644 --- a/python/cuml/cuml/fil/CMakeLists.txt +++ b/python/cuml/cuml/fil/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -27,5 +27,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX fil_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/internals/CMakeLists.txt b/python/cuml/cuml/internals/CMakeLists.txt index d4ebfdd01d..a363fb7323 100644 --- a/python/cuml/cuml/internals/CMakeLists.txt +++ b/python/cuml/cuml/internals/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -26,7 +26,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX internals_ - ASSOCIATED_TARGETS cuml ) # We need to include for callbacks_implements.h in the internals folder diff --git a/python/cuml/cuml/kernel_ridge/CMakeLists.txt b/python/cuml/cuml/kernel_ridge/CMakeLists.txt index 75421a1752..1bf0d0a1e0 100644 --- a/python/cuml/cuml/kernel_ridge/CMakeLists.txt +++ b/python/cuml/cuml/kernel_ridge/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,5 +22,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX kernel_ridge_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/linear_model/CMakeLists.txt b/python/cuml/cuml/linear_model/CMakeLists.txt index aa72642453..cfa0c3ab05 100644 --- a/python/cuml/cuml/linear_model/CMakeLists.txt +++ b/python/cuml/cuml/linear_model/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -37,5 +37,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_mg_libraries}" MODULE_PREFIX linear_model_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/manifold/CMakeLists.txt b/python/cuml/cuml/manifold/CMakeLists.txt index 115705d9af..0ca860afe7 100644 --- a/python/cuml/cuml/manifold/CMakeLists.txt +++ b/python/cuml/cuml/manifold/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -25,5 +25,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX manifold_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/metrics/CMakeLists.txt b/python/cuml/cuml/metrics/CMakeLists.txt index 0a6e789c13..a56575ccb8 100644 --- a/python/cuml/cuml/metrics/CMakeLists.txt +++ b/python/cuml/cuml/metrics/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -27,5 +27,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX metrics_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/metrics/cluster/CMakeLists.txt b/python/cuml/cuml/metrics/cluster/CMakeLists.txt index a81708674c..fbbde707f4 100644 --- a/python/cuml/cuml/metrics/cluster/CMakeLists.txt +++ b/python/cuml/cuml/metrics/cluster/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -28,5 +28,4 @@ rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/neighbors/CMakeLists.txt b/python/cuml/cuml/neighbors/CMakeLists.txt index dbb23550aa..6658ddc5f2 100644 --- a/python/cuml/cuml/neighbors/CMakeLists.txt +++ b/python/cuml/cuml/neighbors/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -32,5 +32,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_mg_libraries}" MODULE_PREFIX neighbors_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/random_projection/CMakeLists.txt b/python/cuml/cuml/random_projection/CMakeLists.txt index f4e54397ac..012382a1bb 100644 --- a/python/cuml/cuml/random_projection/CMakeLists.txt +++ b/python/cuml/cuml/random_projection/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,5 +22,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX random_projection_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/solvers/CMakeLists.txt b/python/cuml/cuml/solvers/CMakeLists.txt index a6eada58a9..ad83508254 100644 --- a/python/cuml/cuml/solvers/CMakeLists.txt +++ b/python/cuml/cuml/solvers/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -31,5 +31,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_mg_libraries}" MODULE_PREFIX solvers_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/svm/CMakeLists.txt b/python/cuml/cuml/svm/CMakeLists.txt index 3b9ab0e199..5d19df1a4a 100644 --- a/python/cuml/cuml/svm/CMakeLists.txt +++ b/python/cuml/cuml/svm/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -25,5 +25,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX svm_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/cuml/tsa/CMakeLists.txt b/python/cuml/cuml/tsa/CMakeLists.txt index 3cbe54bded..92552e9ff4 100644 --- a/python/cuml/cuml/tsa/CMakeLists.txt +++ b/python/cuml/cuml/tsa/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -27,5 +27,4 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${cuml_sg_libraries}" MODULE_PREFIX tsa_ - ASSOCIATED_TARGETS cuml ) diff --git a/python/cuml/pyproject.toml b/python/cuml/pyproject.toml index 820a690d3c..73191ae31f 100644 --- a/python/cuml/pyproject.toml +++ b/python/cuml/pyproject.toml @@ -24,8 +24,8 @@ select = [ "distro-too-large-compressed", ] -# detect when package size grows significantly -max_allowed_size_compressed = '1.5G' +# PyPI limit is 100 MiB, fail CI before we get too close to that +max_allowed_size_compressed = '75M' [tool.pytest.ini_options] addopts = "--tb=native" @@ -99,6 +99,7 @@ dependencies = [ "dask-cuda==25.2.*,>=0.0.0a0", "dask-cudf==25.2.*,>=0.0.0a0", "joblib>=0.11", + "libcuml==25.2.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<3.0a0", "nvidia-cublas", @@ -179,8 +180,11 @@ matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cuda-python", - "cuvs==25.2.*,>=0.0.0a0", "cython>=3.0.0", + "libcuml==25.2.*,>=0.0.0a0", + "libcuvs==25.2.*,>=0.0.0a0", + "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "ninja", "pylibraft==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", diff --git a/python/libcuml/CMakeLists.txt b/python/libcuml/CMakeLists.txt new file mode 100644 index 0000000000..62bc665e4a --- /dev/null +++ b/python/libcuml/CMakeLists.txt @@ -0,0 +1,89 @@ +# ============================================================================= +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +include(rapids-cuda) +rapids_cuda_init_architectures(libcuml-python) + +project( + libcuml-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +# Check if cuml is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python cuml package. +find_package(cuml "${RAPIDS_VERSION}") + +if(cuml_FOUND) + return() +endif() + +unset(cuml_FOUND) + +# --- cuML --- # +set(BUILD_CUML_TESTS OFF) +set(BUILD_PRIMS_TESTS OFF) +set(BUILD_CUML_C_LIBRARY OFF) +set(BUILD_CUML_EXAMPLES OFF) +set(BUILD_CUML_BENCH OFF) + +# In libcuml wheels, we always want to build in all cuML algorithms. +# This is the default in cpp/CMakeLists.txt, but just making that choice for wheels explicit here. +set(CUML_ALGORITHMS "ALL" CACHE STRING "Choose which algorithms are built cuML. Can specify individual algorithms or groups in a semicolon-separated list.") + +# for libcuml wheels, always compile in the multi-node, multi-GPU stuff from cumlprims_mg +set(SINGLEGPU OFF) + +set(CUML_CPP_TARGET "cuml++") +set(CUML_CPP_SRC "../../cpp") + +# --- cumlprims_mg --- # +# ship cumlprims_mg in the 'libcuml' wheel (for re-use by 'cuml' wheels) +set(CUML_USE_CUMLPRIMS_MG_STATIC OFF) +set(CUML_EXCLUDE_CUMLPRIMS_MG_FROM_ALL OFF) + +# --- cuVS --- # +set(CUML_USE_CUVS_STATIC OFF) +set(CUML_EXCLUDE_CUVS_FROM_ALL ON) + +# --- raft --- # +set(CUML_RAFT_CLONE_ON_PIN OFF) +set(CUML_EXCLUDE_RAFT_FROM_ALL ON) + +# --- treelite --- # +set(CUML_EXPORT_TREELITE_LINKAGE ON) +set(CUML_PYTHON_TREELITE_TARGET treelite::treelite_static) +set(CUML_USE_TREELITE_STATIC ON) +set(CUML_EXCLUDE_TREELITE_FROM_ALL ON) + +# --- CUDA --- # +set(CUDA_STATIC_RUNTIME ON) +set(CUDA_STATIC_MATH_LIBRARIES OFF) + +add_subdirectory(../../cpp cuml-cpp) + +# assumes libcuml++ is installed 2 levels deep, e.g. site-packages/cuml/lib64/libcuml++.so +set(rpaths + "$ORIGIN/../../nvidia/cublas/lib" + "$ORIGIN/../../nvidia/cufft/lib" + "$ORIGIN/../../nvidia/curand/lib" + "$ORIGIN/../../nvidia/cusolver/lib" + "$ORIGIN/../../nvidia/cusparse/lib" + "$ORIGIN/../../nvidia/nvjitlink/lib" +) +set_property(TARGET ${CUML_CPP_TARGET} PROPERTY INSTALL_RPATH ${rpaths} APPEND) diff --git a/python/libcuml/LICENSE b/python/libcuml/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libcuml/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libcuml/README.md b/python/libcuml/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libcuml/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libcuml/libcuml/VERSION b/python/libcuml/libcuml/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libcuml/libcuml/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libcuml/libcuml/__init__.py b/python/libcuml/libcuml/__init__.py new file mode 100644 index 0000000000..69d95c8423 --- /dev/null +++ b/python/libcuml/libcuml/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcuml._version import __git_commit__, __version__ +from libcuml.load import load_library diff --git a/python/libcuml/libcuml/_version.py b/python/libcuml/libcuml/_version.py new file mode 100644 index 0000000000..da66c0d576 --- /dev/null +++ b/python/libcuml/libcuml/_version.py @@ -0,0 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcuml/libcuml/load.py b/python/libcuml/libcuml/load.py new file mode 100644 index 0000000000..a3c9d28f4f --- /dev/null +++ b/python/libcuml/libcuml/load.py @@ -0,0 +1,122 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libcuml that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + + Returns ``None`` if the library cannot be loaded. + """ + # cumlprims_mg installs to lib/ + if soname.startswith("libcumlprims_mg"): + relative_libdir = "lib" + else: + relative_libdir = "lib64" + if os.path.isfile( + lib := os.path.join(os.path.dirname(__file__), relative_libdir, soname) + ): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libcuml++.so and its dependencies""" + try: + # libraft must be loaded before libcuml++ because libcuml++ + # references its symbols + import libraft + + libraft.load_library() + except ModuleNotFoundError: + # 'libcuml++' has a runtime dependency on 'libraft'. However, + # that dependency might be satisfied by the 'libraft' conda package + # (which does not have any Python modules), instead of the + # 'libraft' wheel. + # + # In that situation, assume that 'libraft.so' is in a place where + # the loader can find it. + pass + + try: + # libcuvs must be loaded before libcuml++ because libcuml++ + # references its symbols + import libcuvs + + libcuvs.load_library() + except ModuleNotFoundError: + # 'libcuml++' has a runtime dependency on 'libcuvs'. However, + # that dependency might be satisfied by the 'libcuvs' conda package + # (which does not have any Python modules), instead of the + # 'libcuvs' wheel. + # + # In that situation, assume that 'libcuvs.so' is in a place where + # the loader can find it. + pass + + prefer_system_installation = ( + os.getenv("RAPIDS_LIBCUML_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" + ) + + libs_to_return = [] + for soname in ["libcumlprims_mg.so", "libcuml++.so"]: + libcuml_lib = None + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcuml_lib = _load_system_installation(soname) + except OSError: + libcuml_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library was prebuilt before + # packaging the wheel), look for a system installation. + try: + libcuml_lib = _load_wheel_installation(soname) + if libcuml_lib is None: + libcuml_lib = _load_system_installation(soname) + except OSError: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + pass + if libcuml_lib: + libs_to_return.append(libcuml_lib) + + # The caller almost never needs to do anything with these libraries, but no + # harm in offering the option since these objects at least provide handles + # to inspect where libcuml was loaded from. + + return libs_to_return diff --git a/python/libcuml/pyproject.toml b/python/libcuml/pyproject.toml new file mode 100644 index 0000000000..9d1530c1ad --- /dev/null +++ b/python/libcuml/pyproject.toml @@ -0,0 +1,89 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcuml" +dynamic = ["version"] +description = "cuML - RAPIDS ML Algorithms (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.10" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] +dependencies = [ + "libcuvs==25.2.*,>=0.0.0a0", + "libraft==25.2.*,>=0.0.0a0", + "nvidia-cublas", + "nvidia-cufft", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/cuml" + +[project.entry-points."cmake.prefix"] +libcuml= "libcuml" + +[tool.pydistcheck] +select = [ + "distro-too-large-compressed", +] + +# detect when package size grows significantly +max_allowed_size_compressed = '500M' + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libcuml"] +wheel.install-dir = "libcuml" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcuml/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "libcuvs==25.2.*,>=0.0.0a0", + "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", + "ninja", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.