wheels: dynamically load libcudf.so from libcudf wheel (#1447)

Contributes to rapidsai/build-planning#33.

Proposes the following for `cuspatial` wheels:

* add build and runtime dependencies on `libcudf` wheels
* stop vendoring copies of `libcudf.so`, `libnvcomp.so`, `libnvcomp_bitcomp.so`, and `libnvcomp_gdeflate.so`
  - *(load `libcudf.so` dynamically at runtime instead)*

And other related changes for development/CI:

* combine all `pip install` calls into 1 in wheel-testing scripts
  - *like rapidsai/cudf#16575
  - *to improve the chance that packaging issues are discovered in CI*
* `dependencies.yaml` changes:
   - more use of YAML anchors = less duplication
   - use dedicated `depends_on_librmm` and `depends_on_libcudf` groups
* explicitly pass a package type to `gha-tools` wheel uploading/downloading scripts

## Notes for Reviewers

### Benefits of these changes

Unblocks CI in this repo (ref: #1444 (comment), #1441 (comment)).

Reduces wheel sizes for `cuspatial` wheels by about 125MB 😁 

| wheel          | size (before)  | size (this PR) |
|:-----------:|-------------:|---------------:|
| `cuspatial` |   146.0M        |   21M               |
| `cuproj `     |       0.9M       |   0.9M              |
|**TOTAL**   |  **146.9M** | **21.9M**        |

*NOTES: size = compressed, "before" = 2024-08-21 nightlies (c60bd4d), CUDA = 12, Python = 3.11*

<details><summary>how I calculated those (click me)</summary>

```shell
# note: 2024-08-21 because that was the most recent date with
#           successfully-built cuspatial nightlies
#
docker run \
    --rm \
    -v $(pwd):/opt/work:ro \
    -w /opt/work \
    --network host \
    --env RAPIDS_NIGHTLY_DATE=2024-08-21 \
    --env RAPIDS_NIGHTLY_SHA=c60bd4d \
    --env RAPIDS_PR_NUMBER=1447 \
    --env RAPIDS_PY_CUDA_SUFFIX=cu12 \
    --env RAPIDS_REPOSITORY=rapidsai/cuspatial \
    --env WHEEL_DIR_BEFORE=/tmp/wheels-before \
    --env WHEEL_DIR_AFTER=/tmp/wheels-after \
    -it rapidsai/ci-wheel:cuda12.5.1-rockylinux8-py3.11 \
    bash

mkdir -p "${WHEEL_DIR_BEFORE}"
mkdir -p "${WHEEL_DIR_AFTER}"

py_projects=(
    cuspatial
    cuproj
)

for project in "${py_projects[@]}"; do
    # before
    RAPIDS_BUILD_TYPE=nightly \
    RAPIDS_PY_WHEEL_NAME="${project}_${RAPIDS_PY_CUDA_SUFFIX}" \
    RAPIDS_REF_NAME="branch-24.10" \
    RAPIDS_SHA=${RAPIDS_NIGHTLY_SHA} \
        rapids-download-wheels-from-s3 python "${WHEEL_DIR_BEFORE}"

    # after
    RAPIDS_BUILD_TYPE=pull-request \
    RAPIDS_PY_WHEEL_NAME="${project}_${RAPIDS_PY_CUDA_SUFFIX}" \
    RAPIDS_REF_NAME="pull-request/${RAPIDS_PR_NUMBER}" \
        rapids-download-wheels-from-s3 python "${WHEEL_DIR_AFTER}"
done

du -sh ${WHEEL_DIR_BEFORE}/*
du -sh ${WHEEL_DIR_BEFORE}
du -sh ${WHEEL_DIR_AFTER}/*
du -sh ${WHEEL_DIR_AFTER}
```

</details>

Reduces the amount of additional work required to start shipping `libcuspatial` wheels.

### Background

This is part of ongoing work towards packaging `libcuspatial` as a wheel.

relevant prior work:

* packaging `libcudf` wheels: rapidsai/cudf#15483
* consolidating `pip install` calls in CI scripts for `cudf`: rapidsai/cudf#16575
* `cudf` dropping its Arrow library dependency: rapidsai/cudf#16640

### How I tested this

Confirmed in local builds and CI logs that `cudf` is being *found*, not *built*, in `cuspatial` builds.

```text
-- CPM: Using local package [email protected]
```

([build link](https://github.com/rapidsai/cuspatial/actions/runs/10602971716/job/29386288614?pr=1447#step:9:23472))

Built `cuspatial` wheels locally and ran all the unit tests, without issue.

#

Authors:
  - James Lamb (https://github.com/jameslamb)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Vyas Ramasubramani (https://github.com/vyasr)
  - Matthew Roeschke (https://github.com/mroeschke)

URL: #1447

.github/workflows/build.yaml

@@ -85,6 +85,7 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       package-name: cuspatial
+      package-type: python
   wheel-build-cuproj:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -104,3 +105,4 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       package-name: cuproj
+      package-type: python

ci/build_wheel.sh

@@ -5,6 +5,7 @@ set -euo pipefail
 
 package_name=$1
 package_dir=$2
+package_type=$3
 
 source rapids-configure-sccache
 source rapids-date-string
@@ -18,6 +19,9 @@ cd "${package_dir}"
 python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
 
 mkdir -p final_dist
-python -m auditwheel repair -w final_dist dist/*
+python -m auditwheel repair \
+    --exclude libcudf.so \
+    -w final_dist \
+    dist/*
 
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist

ci/build_wheel_cuproj.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
-ci/build_wheel.sh cuproj python/cuproj
+ci/build_wheel.sh cuproj python/cuproj python

ci/build_wheel_cuspatial.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
-export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON"
-
-ci/build_wheel.sh cuspatial python/cuspatial
+ci/build_wheel.sh cuspatial python/cuspatial python

ci/test_wheel_cuproj.sh

@@ -5,19 +5,21 @@ set -eou pipefail
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
-# Install additional dependencies
+# install build dependencies for fiona
 apt update
 DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev
-python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'
 
-# Download the cuspatial built in the previous step
-RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cuspatial-dep
-python -m pip install --no-deps ./local-cuspatial-dep/cuspatial*.whl
+# Download the cuproj and cuspatial built in the previous step
+RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
-python -m pip install $(echo ./dist/cuproj*.whl)[test]
+python -m pip install \
+  --no-binary 'fiona' \
+  "$(echo ./dist/cuspatial*.whl)" \
+  "$(echo ./dist/cuproj*.whl)[test]" \
+  'fiona>=1.8.19,<1.9'
 
 rapids-logger "pytest cuproj"
 pushd python/cuproj/cuproj

ci/test_wheel_cuspatial.sh

@@ -5,15 +5,19 @@ set -eou pipefail
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
-# Install additional dependencies
+# install build dependencies for fiona
 apt update
 DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev
-python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'
+
+# Download the cuspatial built in the previous step
+RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
-python -m pip install $(echo ./dist/cuspatial*.whl)[test]
+python -m pip install \
+  --no-binary 'fiona' \
+  "$(echo ./dist/cuspatial*.whl)[test]" \
+  'fiona>=1.8.19,<1.9'
 
 rapids-logger "pytest cuspatial"
 pushd python/cuspatial/cuspatial

dependencies.yaml

@@ -17,6 +17,8 @@ files:
       - depends_on_cudf
       - depends_on_cuml
       - depends_on_cupy
+      - depends_on_libcudf
+      - depends_on_librmm
       - rapids_build_skbuild
       - run_python_cuspatial
       - test_libcuspatial
@@ -77,14 +79,17 @@ files:
       - build_cpp
       - build_python
       - build_wheels
+      - depends_on_libcudf
+      - depends_on_librmm
   py_run_cuspatial:
     output: [pyproject]
     pyproject_dir: python/cuspatial
     extras:
       table: project
     includes:
-      - depends_on_rmm
       - depends_on_cudf
+      - depends_on_libcudf
+      - depends_on_rmm
       - run_python_cuspatial
   py_test_cuspatial:
     output: [pyproject]
@@ -141,14 +146,12 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - ninja
-          - cmake>=3.26.4,!=3.30.0
+          - &ninja ninja
+          - &cmake cmake>=3.26.4,!=3.30.0
       - output_types: conda
         packages:
           - c-compiler
           - cxx-compiler
-          - libcudf==24.10.*,>=0.0.0a0
-          - librmm==24.10.*,>=0.0.0a0
           - proj
           - sqlite
     specific:
@@ -184,13 +187,13 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - ninja
-          - cmake>=3.26.4,!=3.30.0
+          - *ninja
+          - *cmake
       - output_types: conda
         packages:
           - c-compiler
           - cxx-compiler
-          - librmm==24.10.*,>=0.0.0a0
+          - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0
           - proj
           - sqlite
     specific:
@@ -443,6 +446,31 @@ dependencies:
               - pylibcudf-cu11==24.10.*,>=0.0.0a0
           - {matrix: null, packages: [*cudf_unsuffixed]}
 
+  depends_on_libcudf:
+    common:
+      - output_types: conda
+        packages:
+          - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
+            packages:
+              - libcudf-cu12==24.10.*,>=0.0.0a0
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
+            packages:
+              - libcudf-cu11==24.10.*,>=0.0.0a0
+          - {matrix: null, packages: [*libcudf_unsuffixed]}
+
   depends_on_cuml:
     common:
       - output_types: conda
@@ -523,3 +551,28 @@ dependencies:
           - libcuspatial==24.10.*,>=0.0.0a0
           - cuspatial==24.10.*,>=0.0.0a0
           - cuproj==24.10.*,>=0.0.0a0
+
+  depends_on_librmm:
+    common:
+      - output_types: conda
+        packages:
+          - *librmm_unsuffixed
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
+            packages:
+              - librmm-cu12==24.10.*,>=0.0.0a0
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
+            packages:
+              - librmm-cu11==24.10.*,>=0.0.0a0
+          - {matrix: null, packages: [*librmm_unsuffixed]}

python/cuspatial/CMakeLists.txt

@@ -43,15 +43,7 @@ if(NOT cuspatial_FOUND)
   add_subdirectory(../../cpp cuspatial-cpp EXCLUDE_FROM_ALL)
 
   set(cython_lib_dir cuspatial)
-  include(cmake/Modules/WheelHelpers.cmake)
-  # TODO: This install is currently overzealous. We should only install the libraries that are
-  # downloaded by CPM during the build, not libraries that were found on the system.  However, in
-  # practice this would only be a problem if libcudf was not found but some of the
-  # dependencies were, and we have no real use cases where that happens.
-  install_aliased_imported_targets(
-    TARGETS cuspatial arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp
-    DESTINATION ${cython_lib_dir}
-  )
+  install(TARGETS cuspatial DESTINATION ${cython_lib_dir})
 endif()
 
 include(rapids-cython-core)

python/cuspatial/cuspatial/__init__.py

@@ -1,4 +1,8 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+# load cudf before any other imports, to be sure libcudf's symbols are found
+# in the libcudf.so from the libcudf wheel (if one is installed)
+import cudf
 
 from ._version import __git_commit__, __version__
 from .core.geodataframe import GeoDataFrame

python/cuspatial/pyproject.toml

@@ -32,6 +32,7 @@ requires-python = ">=3.10"
 dependencies = [
     "cudf==24.10.*,>=0.0.0a0",
     "geopandas>=0.11.0",
+    "libcudf==24.10.*,>=0.0.0a0",
     "numpy>=1.23,<2.0a0",
     "rmm==24.10.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
@@ -71,6 +72,7 @@ known_dask = [
 known_rapids = [
     "rmm",
     "cudf",
+    "libcudf",
     "pylibcudf",
 ]
 known_first_party = [
@@ -134,6 +136,8 @@ requires = [
     "cmake>=3.26.4,!=3.30.0",
     "cudf==24.10.*,>=0.0.0a0",
     "cython>=3.0.0",
+    "libcudf==24.10.*,>=0.0.0a0",
+    "librmm==24.10.*,>=0.0.0a0",
     "ninja",
     "rmm==24.10.*,>=0.0.0a0",
     "wheel",