Use libkvikio conda packages in libcudf, add explicit libcufile depen…

…dency. (#13231)

Provides a partial solution to #13230. During conda builds of `libcudf`, `libkvikio` is currently being fetched by CPM rather than supplied by a conda dependency. This means `libkvikio` headers are being shipped as part of `libcudf`'s packages, which is not desirable.

I also added `libcufile[-dev]` as an explicit dependency of libcudf. Note that this is only available on linux64 (amd64), not aarch64 (arm64). We should always make the cuFile library available at build time for conda packages on amd64.

The impacts of this change are:
- `libcudf` conda packages will no longer ship `libkvikio` headers (those headers will instead be supplied by `libkvikio` packages). This reduces the size of libcudf and prevents clobbering files from `libkvikio`.
- `libcudf` will have a dependency on `libcufile` from the `nvidia` channel on `linux64` (but not `aarch64`, since libcufile packages currently do not exist for `aarch64`).

This change does not impact builds outside of conda-build.

Authors:
  - Bradley Dice (https://github.com/bdice)
  - Vukasin Milovanovic (https://github.com/vuule)

Approvers:
  - AJ Schmidt (https://github.com/ajschmidt8)
  - Vukasin Milovanovic (https://github.com/vuule)

URL: #13231

ci/release/update-version.sh

@@ -78,10 +78,11 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/dask_cudf/so
 
 # bump rmm & dask-cuda
 for FILE in conda/environments/*.yaml dependencies.yaml; do
-  sed_runner "s/dask-cuda==${CURRENT_SHORT_TAG_PEP440}/dask-cuda==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
-  sed_runner "s/rmm==${CURRENT_SHORT_TAG_PEP440}/rmm==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
   sed_runner "s/cudf==${CURRENT_SHORT_TAG_PEP440}/cudf==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
   sed_runner "s/cudf_kafka==${CURRENT_SHORT_TAG_PEP440}/cudf_kafka==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/dask-cuda==${CURRENT_SHORT_TAG_PEP440}/dask-cuda==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/kvikio==${CURRENT_SHORT_TAG_PEP440}/kvikio==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/rmm==${CURRENT_SHORT_TAG_PEP440}/rmm==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
 done
 
 # Doxyfile update

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -38,6 +38,7 @@ dependencies:
 - libarrow==11.0.0.*
 - libcurand-dev=10.3.0.86
 - libcurand=10.3.0.86
+- libkvikio==23.6.*
 - librdkafka>=1.9.0,<1.10.0a0
 - librmm==23.6.*
 - mimesis>=4.1.0

conda/recipes/libcudf/conda_build_config.yaml

@@ -25,6 +25,16 @@ dlpack_version:
 librdkafka_version:
   - ">=1.9.0,<1.10.0a0"
 
+# The CTK libraries below are missing from the conda-forge::cudatoolkit
+# package. The "*_host_*" version specifiers correspond to `11.8` packages and the
+# "*_run_*" version specifiers correspond to `11.x` packages.
+
+libcufile_host_version:
+  - "1.4.0.31"
+
+libcufile_run_version:
+  - ">=1.0.0.82,<=1.4.0.31"
+
 libcurand_host_version:
   - "=10.3.0.86"
 

conda/recipes/libcudf/meta.yaml

@@ -42,10 +42,13 @@ requirements:
     - sysroot_{{ target_platform }} {{ sysroot_version }}
   host:
     - librmm ={{ minor_version }}
+    - libkvikio ={{ minor_version }}
     - cudatoolkit ={{ cuda_version }}
     - cuda-nvrtc ={{ cuda_version }}
     - cuda-nvrtc-dev ={{ cuda_version }}
     - cuda-nvtx ={{ cuda_version }}
+    - libcufile {{ libcufile_host_version }}      # [linux64]
+    - libcufile-dev {{ libcufile_host_version }}  # [linux64]
     - libcurand {{ libcurand_host_version }}
     - libcurand-dev {{ libcurand_host_version }}
     - libarrow {{ libarrow_version }}
@@ -73,7 +76,10 @@ outputs:
       run:
         - cudatoolkit {{ cuda_spec }}
         - librmm ={{ minor_version }}
+        - libkvikio ={{ minor_version }}
         - libarrow {{ libarrow_version }}
+        - libcufile {{ libcufile_run_version }}      # [linux64]
+        - libcufile-dev {{ libcufile_run_version }}  # [linux64]
         - dlpack {{ dlpack_version }}
         - gtest {{ gtest_version }}
         - gmock {{ gtest_version }}

dependencies.yaml

@@ -209,6 +209,7 @@ dependencies:
       - output_types: [conda, requirements]
         packages:
           - librmm==23.6.*
+          - libkvikio==23.6.*
       - output_types: conda
         packages:
           - fmt>=9.1.0,<10