adding wheel build for libcudf

.github/workflows/pr.yaml

@@ -23,8 +23,9 @@ jobs:
       - static-configure
       - conda-notebook-tests
       - docs-build
-      - wheel-build-cudf
-      - wheel-tests-cudf
+      - wheel-build-cpp
+      - wheel-build-python
+      - wheel-tests-python
       - wheel-build-dask-cudf
       - wheel-tests-dask-cudf
       - devcontainer
@@ -118,22 +119,30 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/build_docs.sh"
-  wheel-build-cudf:
+  wheel-build-cpp:
     needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
       build_type: pull-request
-      script: "ci/build_wheel_cudf.sh"
-  wheel-tests-cudf:
-    needs: wheel-build-cudf
+      script: "ci/build_wheel_cpp.sh"
+  wheel-build-python:
+    needs: [checks, wheel-build-cpp]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: "ci/build_wheel_python.sh"
+  wheel-tests-python:
+    needs: wheel-build-python
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/test_wheel_cudf.sh
   wheel-build-dask-cudf:
-    needs: wheel-build-cudf
+    needs: wheel-build-python
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
@@ -161,7 +170,7 @@ jobs:
         build-all -DBUILD_BENCHMARKS=ON --verbose;
         sccache -s;
   unit-tests-cudf-pandas:
-    needs: wheel-build-cudf
+    needs: wheel-build-python
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
@@ -170,7 +179,7 @@ jobs:
       script: ci/cudf_pandas_scripts/run_tests.sh
   pandas-tests:
     # run the Pandas unit tests using PR branch
-    needs: wheel-build-cudf
+    needs: wheel-build-python
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:

build.sh

@@ -17,12 +17,13 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd $(dirname $0); pwd)
 
-VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats"
-HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
+VALIDARGS="clean libcudf cudf libcudfwheel cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats"
+HELP="$0 [clean] [libcudf] [cudf] [libcudfwheel] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
    clean                         - remove all existing build artifacts and configuration (start
                                    over)
    libcudf                       - build the cudf C++ code only
    cudf                          - build the cudf Python package
+   libcudfwheel                  - build the cudf C++ code packaged as a python wheel package
    cudfjar                       - build cudf JAR with static libcudf using devtoolset toolchain
    dask_cudf                     - build the dask_cudf Python package
    benchmarks                    - build benchmarks
@@ -333,7 +334,14 @@ if buildAll || hasArg libcudf; then
     fi
 fi
 
-# Build and install the cudf Python package
+if buildAll || hasArg libcudfwheel; then
+
+    cd ${REPODIR}/python/libcudf
+    SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};-DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES};${EXTRA_CMAKE_ARGS}" \
+        python ${PYTHON_ARGS_FOR_INSTALL} .
+fi
+
+# Build and install the cudf Python packages
 if buildAll || hasArg cudf; then
 
     cd ${REPODIR}/python/cudf

ci/build_wheel_cpp.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_dir="python/libcudf"
+
+./ci/build_wheel.sh ${package_dir}
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+mkdir -p ${package_dir}/final_dist
+python -m auditwheel repair --exclude libarrow.so.1601 -w ${package_dir}/final_dist ${package_dir}/dist/*
+
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist

ci/build_wheel_python.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="cudf"
+package_dir="python/cudf"
+
+export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON"
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+# Downloads libcudf wheel from this current build, then points pip to it in PIP_FIND_LINKS below
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist
+
+export PIP_FIND_LINKS="/tmp/libcudf_dist"
+./ci/build_wheel.sh ${package_dir}
+
+cd ${package_dir}
+mkdir -p final_dist
+python -m auditwheel repair --exclude libcudf.so --exclude libarrow.so.1601 -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist

ci/cudf_pandas_scripts/pandas-tests/run.sh

@@ -11,8 +11,9 @@ rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids
 rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}"
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep
-python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests]
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep
+RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep
+python -m pip install --find-links $(pwd)/local-cudf-dep $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests]
 
 RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"}
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/

ci/cudf_pandas_scripts/run_tests.sh

@@ -36,8 +36,9 @@ if [ "$no_cudf" = true ]; then
     echo "Skipping cudf install"
 else
     RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-    RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep
-    python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests]
+    RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep
+    RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep
+    python -m pip install --find-links $(pwd)/local-cudf-dep $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests]
 fi
 
 python -m pytest -p cudf.pandas \

ci/test_wheel_cudf.sh

@@ -4,10 +4,11 @@
 set -eou pipefail
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
+RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
-python -m pip install $(echo ./dist/cudf*.whl)[test]
+python -m pip install --find-links $(pwd)/dist $(echo ./dist/cudf*.whl)[test]
 
 RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"}
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/

ci/test_wheel_dask_cudf.sh

@@ -7,8 +7,9 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
 
 # Download the cudf built in the previous step
-RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep
-python -m pip install --no-deps ./local-cudf-dep/cudf*.whl
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep
+RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep
+python -m pip install --no-deps --find-links $(pwd)/local-cudf-dep ./local-cudf-dep/cudf*.whl
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install $(echo ./dist/dask_cudf*.whl)[test]

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -40,6 +40,7 @@ dependencies:
 - libarrow-acero==16.1.0.*
 - libarrow-dataset==16.1.0.*
 - libarrow==16.1.0.*
+- libcudf==24.8.*,>=0.0.0a0
 - libcufile-dev=1.4.0.31
 - libcufile=1.4.0.31
 - libcurand-dev=10.3.0.86

conda/environments/all_cuda-122_arch-x86_64.yaml

@@ -41,6 +41,7 @@ dependencies:
 - libarrow-acero==16.1.0.*
 - libarrow-dataset==16.1.0.*
 - libarrow==16.1.0.*
+- libcudf==24.8.*,>=0.0.0a0
 - libcufile-dev
 - libcurand-dev
 - libkvikio==24.8.*

cpp/CMakeLists.txt

@@ -685,6 +685,15 @@ set_source_files_properties(
   src/transform/transform.cpp
   PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64"
 )
+# We need to hide symbols in the logger TU by default because those symbols include spdlog symbols
+# that are compiled in because we use it as a header-only library. This is problematic because other
+# libraries that also do the same thing will have their own copies, and they will all be weak
+# symbols so which one is used is undefined. Marking these symbols as hidden makes them local and
+# ensures that internal calls to spdlog functions correctly use the local copy.
+set_source_files_properties(
+  src/utilities/logger.cpp PROPERTIES COMPILE_OPTIONS
+                                      "-fvisibility=hidden;-fvisibility-inlines-hidden"
+)
 
 set_property(
   SOURCE src/io/parquet/writer_impl.cu

cpp/include/cudf/utilities/logger.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -41,6 +41,11 @@ namespace cudf {
  *
  * @return spdlog::logger& The logger.
  */
-spdlog::logger& logger();
+// This is a workaround for Doxygen not being able to parse the `visibility` attribute
+/// \cond DOXYGEN_IGNORE
+// We need this symbol to be visible because this TU is marked as having all
+// symbols hidden by default in cpp/CMakelists.txt.
+__attribute__((visibility("default"))) spdlog::logger& logger();
+/// \endcond
 
 }  // namespace cudf

dependencies.yaml

@@ -22,6 +22,7 @@ files:
       - rapids_build_setuptools
       - run_common
       - run_cudf
+      - run_libcudf
       - run_dask_cudf
       - run_custreamz
       - test_cpp
@@ -76,13 +77,33 @@ files:
       - docs
       - libarrow_run
       - py_version
+  # This is the shared library, bundled as a wheel. It is meant to be consumed by the wrapper.
+  py_rapids_build_libcudf:
+    output: pyproject
+    pyproject_dir: python/libcudf
+    extras:
+      table: build-system
+    includes:
+      - rapids_build_skbuild
+  # This is the wrapper that gets used in Python, not the shared library wheel
   py_rapids_build_cudf:
     output: pyproject
     pyproject_dir: python/cudf
     extras:
       table: build-system
     includes:
       - rapids_build_skbuild
+  py_build_libcudf:
+    output: pyproject
+    pyproject_dir: python/libcudf
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes:
+      - build_base
+      - build_cpp
+      - build_python_common
+      - build_python_cudf
   py_build_cudf:
     output: pyproject
     pyproject_dir: python/cudf
@@ -93,6 +114,14 @@ files:
       - build_base
       - build_python_common
       - build_python_cudf
+      - run_libcudf
+  py_run_libcudf:
+    output: pyproject
+    pyproject_dir: python/libcudf
+    extras:
+      table: project
+    includes:
+      - pyarrow_run
   py_run_cudf:
     output: pyproject
     pyproject_dir: python/cudf
@@ -103,6 +132,7 @@ files:
       - run_cudf
       - pyarrow_run
       - depends_on_cupy
+      - run_libcudf
   py_test_cudf:
     output: pyproject
     pyproject_dir: python/cudf
@@ -374,6 +404,21 @@ dependencies:
         packages:
           # Allow runtime version to float up to patch version
           - pyarrow>=16.1.0,<16.2.0a0
+  run_libcudf:
+    common:
+      - output_types: [conda, requirements]
+        packages:
+          - &libcudf_conda libcudf==24.8.*,>=0.0.0a0
+    specific:
+      - output_types: [pyproject]
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - libcudf-cu12==24.8.*,>=0.0.0a0
+          - matrix: {cuda: "11.*"}
+            packages:
+              - libcudf-cu11==24.8.*,>=0.0.0a0
+          - {matrix: null, packages: [*libcudf_conda] }
   cuda_version:
     specific:
       - output_types: conda