Add libucxx wheel (#260)

Contributes to rapidsai/build-planning#33

Authors:
  - Kyle Edwards (https://github.com/KyleFromNVIDIA)

Approvers:
  - Peter Andreas Entschev (https://github.com/pentschev)
  - Bradley Dice (https://github.com/bdice)

URL: #260

.github/workflows/build.yaml

@@ -57,7 +57,27 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-build-libucxx:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_ucxx.sh
+  wheel-publish-libucxx:
+    needs: wheel-build-libucxx
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libucxx
   wheel-build-ucxx:
+    needs: wheel-build-libucxx
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:

.github/workflows/pr.yaml

@@ -18,6 +18,7 @@ jobs:
       - conda-cpp-tests
       - conda-python-tests
       - conda-python-distributed-tests
+      - wheel-build-libucxx
       - wheel-build-ucxx
       - wheel-tests-ucxx
       - wheel-build-distributed-ucxx
@@ -67,10 +68,17 @@ jobs:
       build_type: pull-request
       script: "ci/test_python_distributed.sh"
       container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"
-  wheel-build-ucxx:
+  wheel-build-libucxx:
     needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_libucxx.sh
+  wheel-build-ucxx:
+    needs: wheel-build-libucxx
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_ucxx.sh

ci/build_wheel.sh

@@ -16,14 +16,25 @@ rapids-generate-version > ./VERSION
 if [[ ${package_name} == "distributed-ucxx" ]]; then
     python -m pip wheel "${package_dir}/" -w "${package_dir}/dist" -vvv --no-deps --disable-pip-version-check
 
-    RAPIDS_PY_WHEEL_NAME="distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/dist
-elif [[ ${package_name} == "ucxx" ]]; then
-    SKBUILD_CMAKE_ARGS="-DUCXX_ENABLE_RMM=ON;-DCMAKE_INSTALL_LIBDIR=ucxx/lib64;-DCMAKE_INSTALL_INCLUDEDIR=ucxx/include" \
+    RAPIDS_PY_WHEEL_NAME="distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${package_dir}/dist
+elif [[ ${package_name} == "libucxx" ]]; then
+    SKBUILD_CMAKE_ARGS="-DUCXX_ENABLE_RMM=ON" \
         python -m pip wheel "${package_dir}"/ -w "${package_dir}"/dist -vvv --no-deps --disable-pip-version-check
 
     python -m auditwheel repair -w ${package_dir}/final_dist --exclude "libucp.so.0" ${package_dir}/dist/*
 
-    RAPIDS_PY_WHEEL_NAME="ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist
+    RAPIDS_PY_WHEEL_NAME="libucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist
+elif [[ ${package_name} == "ucxx" ]]; then
+    CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libucxx_dist)
+    echo "libucxx-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/libucxx_*.whl)" > "${package_dir}/constraints.txt"
+
+    PIP_CONSTRAINT="${package_dir}/constraints.txt" \
+    SKBUILD_CMAKE_ARGS="-DFIND_UCXX_CPP=ON;-DCMAKE_INSTALL_LIBDIR=ucxx/lib64;-DCMAKE_INSTALL_INCLUDEDIR=ucxx/include" \
+        python -m pip wheel "${package_dir}"/ -w "${package_dir}"/dist -vvv --no-deps --disable-pip-version-check
+
+    python -m auditwheel repair -w ${package_dir}/final_dist --exclude "libucp.so.0" --exclude "libucxx.so" ${package_dir}/dist/*
+
+    RAPIDS_PY_WHEEL_NAME="ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${package_dir}/final_dist
 else
   echo "Unknown package '${package_name}'"
   exit 1

ci/build_wheel_libucxx.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_dir="python/libucxx"
+
+./ci/build_wheel.sh libucxx ${package_dir}

ci/release/update-version.sh

@@ -46,6 +46,7 @@ DEPENDENCIES=(
   rmm
 )
 UCXX_DEPENDENCIES=(
+  libucxx
   ucxx
   distributed-ucxx
 )

ci/test_wheel_distributed_ucxx.sh

@@ -9,11 +9,13 @@ source "$(dirname "$0")/test_common.sh"
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
-ucxx_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-ucxx-dep)
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+ucxx_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-ucxx-dep)
+libucxx_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libucxx-dep)
 
 python -m pip install \
     -v \
+    "${libucxx_wheelhouse}"/libucxx_${RAPIDS_PY_CUDA_SUFFIX}*.whl \
     "${ucxx_wheelhouse}"/ucxx_${RAPIDS_PY_CUDA_SUFFIX}*.whl \
     "$(echo ./dist/${package_name}_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
 

ci/test_wheel_ucxx.sh

@@ -9,8 +9,13 @@ source "$(dirname "$0")/test_common.sh"
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
-python -m pip install -v "$(echo ./dist/${package_name}_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+libucxx_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="libucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libucxx-dep)
+
+python -m pip install \
+    -v \
+    "${libucxx_wheelhouse}"/libucxx_${RAPIDS_PY_CUDA_SUFFIX}*.whl \
+    "$(echo ./dist/${package_name}_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
 
 rapids-logger "Python Core Tests"
 run_py_tests

dependencies.yaml

@@ -56,32 +56,57 @@ files:
     includes:
       - docs
       - py_version
-  py_rapids_build_ucxx:
+  py_build_libucxx:
     output: pyproject
-    pyproject_dir: python/ucxx
+    pyproject_dir: python/libucxx
     extras:
       table: build-system
     includes:
       - rapids_build_skbuild
+  py_rapids_build_libucxx:
+    output: pyproject
+    pyproject_dir: python/libucxx
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes:
+      - build_cpp
+      - depends_on_librmm
+      - depends_on_ucx_build
+  py_run_libucxx:
+    output: pyproject
+    pyproject_dir: python/libucxx
+    extras:
+      table: project
+    includes:
+      - depends_on_ucx_run
   py_build_ucxx:
+    output: pyproject
+    pyproject_dir: python/ucxx
+    extras:
+      table: build-system
+    includes:
+      - rapids_build_skbuild
+  py_rapids_build_ucxx:
     output: pyproject
     pyproject_dir: python/ucxx
     extras:
       table: tool.rapids-build-backend
       key: requires
     includes:
       - build_python_ucxx
+      - depends_on_librmm
+      - depends_on_libucxx
       - depends_on_rmm
-      - depends_on_ucx_build
   py_run_ucxx:
     output: pyproject
     pyproject_dir: python/ucxx
     extras:
       table: project
     includes:
       - run_python_ucxx
+      - depends_on_libucxx
       - depends_on_rmm
-      - depends_on_ucx_run
   py_test_ucxx:
     output: pyproject
     pyproject_dir: python/ucxx
@@ -135,6 +160,10 @@ dependencies:
           - librmm==24.10.*,>=0.0.0a0
           - ninja
           - spdlog>=1.12.0,<1.13
+      - output_types: [requirements, pyproject]
+        packages:
+          - *cmake_ver
+          - ninja
   build_python_ucxx:
     common:
       - output_types: [conda, requirements, pyproject]
@@ -298,6 +327,26 @@ dependencies:
             packages: &cupy_packages_cu11
               - cupy-cuda11x>=12.0.0
           - {matrix: null, packages: *cupy_packages_cu11}
+  depends_on_librmm:
+    common:
+      - output_types: conda
+        packages:
+          - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - librmm-cu12==24.10.*,>=0.0.0a0
+          - matrix: {cuda: "11.*"}
+            packages:
+              - librmm-cu11==24.10.*,>=0.0.0a0
+          - {matrix: null, packages: [*librmm_unsuffixed]}
   depends_on_rmm:
     common:
       - output_types: conda
@@ -346,6 +395,23 @@ dependencies:
             packages:
               - cudf-cu11==24.10.*,>=0.0.0a0
           - {matrix: null, packages: [*cudf_unsuffixed]}
+  depends_on_libucxx:
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
+            packages:
+              - libucxx-cu12==0.40.*,>=0.0.0a0
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
+            packages:
+              - libucxx-cu11==0.40.*,>=0.0.0a0
+          - matrix:
+            packages:
+              - libucxx==0.40.*,>=0.0.0a0
   depends_on_ucxx:
     common:
       - output_types: conda

python/libucxx/CMakeLists.txt

@@ -0,0 +1,50 @@
+# =============================================================================
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
+
+file(READ "${CMAKE_CURRENT_LIST_DIR}/../../VERSION" _version_contents)
+if(_version_contents MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*$")
+  set(ucxx_version "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}")
+else()
+  string(REPLACE "\n" "\n  " _version_contents_formatted "${_version_contents}")
+  message(FATAL_ERROR "Could not determine ucxx version. Contents of VERSION file:\n  ${_version_contents_formatted}")
+endif()
+
+include(../../fetch_rapids.cmake)
+include(rapids-cpm)
+include(rapids-cuda)
+rapids_cuda_init_architectures(libucxx-python)
+rapids_cpm_init()
+
+project(
+  libucxx-python
+  VERSION "${ucxx_version}"
+  LANGUAGES CXX
+)
+
+# Check if ucxx is already available. If so, it's the user's responsibility to ensure that the
+# CMake package is also available at build time of the Python ucxx package.
+find_package(ucxx "${ucxx_version}")
+
+if(ucxx_FOUND)
+  return()
+endif()
+
+unset(ucxx_FOUND)
+
+set(BUILD_TESTS OFF)
+set(BUILD_BENCHMARKS OFF)
+
+add_subdirectory(../../cpp ucxx-cpp)

python/libucxx/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

python/libucxx/README.md

@@ -0,0 +1 @@
+../../README.md

python/libucxx/libucxx/VERSION

@@ -0,0 +1 @@
+../../../VERSION

python/libucxx/libucxx/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from libucxx._version import __git_commit__, __version__
+from libucxx.load import load_library
+
+__all__ = ["load_library", "__git_commit__", "__version__"]

python/libucxx/libucxx/_version.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.resources
+
+__version__ = (
+    importlib.resources.files(__package__).joinpath("VERSION").read_text().strip()
+)
+try:
+    __git_commit__ = (
+        importlib.resources.files(__package__)
+        .joinpath("GIT_COMMIT")
+        .read_text()
+        .strip()
+    )
+except FileNotFoundError:
+    __git_commit__ = ""
+
+__all__ = ["__git_commit__", "__version__"]