nest cuml one level deeper in python (#5944)

This is a step towards adding support for dynamic linking with wheels (splitting the shared libraries out into their own wheels). That's being tracked in rapidsai/build-planning#33

This PR performs a necessary step of moving the cuml folder one level deeper, such that the python folder becomes a parent of multiple full-fledged projects, instead of having the python folder be the top level of one python project. This is split into this PR because this change affects so many files. It's easier to review the actual changes of supporting the split wheel when you don't have to also consider these moves.

This change also affects devcontainers, and there will need to be a change similar to rapidsai/devcontainers#283 for cuml.

Authors:
  - Mike Sarahan (https://github.com/msarahan)
  - Kyle Edwards (https://github.com/KyleFromNVIDIA)

Approvers:
  - James Lamb (https://github.com/jameslamb)
  - Dante Gama Dessavre (https://github.com/dantegd)

URL: #5944

.devcontainer/cuda11.8-pip/devcontainer.json

@@ -5,7 +5,7 @@
     "args": {
       "CUDA": "11.8",
       "PYTHON_PACKAGE_MANAGER": "pip",
-      "BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-ucx1.15.0-openmpi-ubuntu22.04"
+      "BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04"
     }
   },
   "runArgs": [

.devcontainer/cuda12.2-pip/devcontainer.json

@@ -5,7 +5,7 @@
     "args": {
       "CUDA": "12.2",
       "PYTHON_PACKAGE_MANAGER": "pip",
-      "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.2-ucx1.15.0-openmpi-ubuntu22.04"
+      "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.2-ucx1.17.0-openmpi-ubuntu22.04"
     }
   },
   "runArgs": [

.github/CODEOWNERS

@@ -5,11 +5,11 @@ cpp/               @rapidsai/cuml-cpp-codeowners
 python/            @rapidsai/cuml-python-codeowners
 
 #cmake code owners
-**/CMakeLists.txt  @rapidsai/cuml-cmake-codeowners
-**/cmake/          @rapidsai/cuml-cmake-codeowners
-python/setup.py    @rapidsai/cuml-cmake-codeowners
-build.sh           @rapidsai/cuml-cmake-codeowners
-**/build.sh        @rapidsai/cuml-cmake-codeowners
+**/CMakeLists.txt          @rapidsai/cuml-cmake-codeowners
+**/cmake/                  @rapidsai/cuml-cmake-codeowners
+python/cuml/pyproject.toml @rapidsai/cuml-cmake-codeowners
+build.sh                   @rapidsai/cuml-cmake-codeowners
+**/build.sh                @rapidsai/cuml-cmake-codeowners
 
 #CI code owners
 /.github/                @rapidsai/ci-codeowners

.pre-commit-config.yaml

@@ -7,12 +7,12 @@ repos:
       hooks:
           - id: black
             files: python/.*
-            args: [--config, python/pyproject.toml]
+            args: [--config, python/cuml/pyproject.toml]
     - repo: https://github.com/PyCQA/flake8
       rev: 5.0.4
       hooks:
           - id: flake8
-            args: [--config=python/.flake8]
+            args: [--config=python/cuml/.flake8]
             files: python/.*$
             types: [file]
             types_or: [python, cython]

build.sh

@@ -61,7 +61,7 @@ HELP="$0 [<target> ...] [<flag> ...]
    CUML_EXTRA_PYTHON_ARGS - Extra arguments to pass directly to pip install
 "
 LIBCUML_BUILD_DIR=${LIBCUML_BUILD_DIR:=${REPODIR}/cpp/build}
-CUML_BUILD_DIR=${REPODIR}/python/build
+CUML_BUILD_DIR=${REPODIR}/python/cuml/build
 PYTHON_DEPS_CLONE=${REPODIR}/python/external_repositories
 BUILD_DIRS="${LIBCUML_BUILD_DIR} ${CUML_BUILD_DIR} ${PYTHON_DEPS_CLONE}"
 
@@ -294,7 +294,7 @@ if (! hasArg --configure-only) && (completeBuild || hasArg cuml || hasArg pydocs
     fi
 
     SKBUILD_CMAKE_ARGS="-DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL};${SKBUILD_EXTRA_CMAKE_ARGS}" \
-        python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python
+        python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python/cuml
 
     if hasArg pydocs; then
         cd ${REPODIR}/docs
@@ -304,5 +304,5 @@ fi
 
 if hasArg cuml-cpu; then
     SKBUILD_CMAKE_ARGS="-DCUML_CPU=ON;-DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE" \
-        python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python
+        python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python/cuml
 fi

ci/build_wheel.sh

@@ -3,7 +3,7 @@
 
 set -euo pipefail
 
-package_dir="python"
+package_dir="python/cuml"
 
 source rapids-configure-sccache
 source rapids-date-string

ci/release/update-version.sh

@@ -38,10 +38,10 @@ function sed_runner() {
 echo "${NEXT_FULL_TAG}" > VERSION
 
 # pyproject.toml versions
-sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/pyproject.toml
-sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/pyproject.toml
-sed_runner "s/pylibraft==.*\",/pylibraft==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/pyproject.toml
-sed_runner "s/raft-dask==.*\",/raft-dask==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/pyproject.toml
+sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/cuml/pyproject.toml
+sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/cuml/pyproject.toml
+sed_runner "s/pylibraft==.*\",/pylibraft==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/cuml/pyproject.toml
+sed_runner "s/raft-dask==.*\",/raft-dask==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/cuml/pyproject.toml
 
 DEPENDENCIES=(
   cudf
@@ -66,7 +66,7 @@ for FILE in dependencies.yaml conda/environments/*.yaml; do
 done
 
 sed_runner "s|/branch-[^/]*/|/branch-${NEXT_SHORT_TAG}/|g" README.md
-sed_runner "s|/branch-[^/]*/|/branch-${NEXT_SHORT_TAG}/|g" python/README.md
+sed_runner "s|/branch-[^/]*/|/branch-${NEXT_SHORT_TAG}/|g" python/cuml/README.md
 
 
 # CI files

ci/run_cuml_dask_pytests.sh

@@ -2,7 +2,7 @@
 # Copyright (c) 2024, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_dask_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/tests/dask
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/dask
 
 rapids-logger "pytest cuml-dask (No UCX-Py/UCXX)"
 timeout 2h python -m pytest --cache-clear "$@" .

ci/run_cuml_integration_pytests.sh

@@ -2,6 +2,6 @@
 # Copyright (c) 2024, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/tests
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests
 
 python -m pytest -p cudf.pandas --cache-clear --ignore=dask -m "not memleak" "$@" --quick_run .

ci/run_cuml_singlegpu_memleak_pytests.sh

@@ -2,6 +2,6 @@
 # Copyright (c) 2024, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/tests
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests
 
 python -m pytest --cache-clear --ignore=dask -m "memleak" "$@" .

ci/run_cuml_singlegpu_pytests.sh

@@ -2,6 +2,6 @@
 # Copyright (c) 2024, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/tests
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests
 
 python -m pytest --cache-clear --ignore=dask -m "not memleak" "$@" .

dependencies.yaml

@@ -68,12 +68,14 @@ files:
       - test_notebooks
   py_build:
     output: pyproject
+    pyproject_dir: python/cuml
     extras:
       table: build-system
     includes:
       - rapids_build_backend
   py_rapids_build:
     output: pyproject
+    pyproject_dir: python/cuml
     extras:
       table: tool.rapids-build-backend
       key: requires
@@ -82,12 +84,14 @@ files:
       - py_build
   py_run:
     output: pyproject
+    pyproject_dir: python/cuml
     extras:
       table: project
     includes:
       - py_run
   py_test:
     output: pyproject
+    pyproject_dir: python/cuml
     extras:
       table: project.optional-dependencies
       key: test

docs/source/conf.py

@@ -201,7 +201,7 @@ def setup(app):
 # The following is used by sphinx.ext.linkcode to provide links to github
 linkcode_resolve = make_linkcode_resolve(
     'cuml', 'https://github.com/rapidsai/'
-    'cuml/blob/{revision}/python/'
+    'cuml/blob/{revision}/python/cuml/'
     '{package}/{path}#L{lineno}')
 
 # Set the default role for interpreted code (anything surrounded in `single

python/CMakeLists.txt → python/cuml/CMakeLists.txt

@@ -14,7 +14,7 @@
 
 cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
 
-include(../rapids_config.cmake)
+include(../../rapids_config.cmake)
 
 option(CUML_CPU "Build only cuML CPU Python components." OFF)
 set(language_list "CXX")
@@ -50,7 +50,7 @@ message(VERBOSE "CUML_PY: Disabling all mnmg components and comms libraries: ${S
 set(CUML_ALGORITHMS "ALL" CACHE STRING "Choose which algorithms are built cuML. Can specify individual algorithms or groups in a semicolon-separated list.")
 
 set(CUML_CPP_TARGET "cuml++")
-set(CUML_CPP_SRC "../cpp")
+set(CUML_CPP_SRC "../../cpp")
 
 ################################################################################
 # - Process User Options  ------------------------------------------------------
@@ -63,7 +63,7 @@ if(FIND_CUML_CPP)
   include(rapids-export)
   rapids_cpm_init()
   find_package(cuml ${CUML_VERSION} REQUIRED)
-  include(../cpp/cmake/thirdparty/get_treelite.cmake)
+  include(${CUML_CPP_SRC}/cmake/thirdparty/get_treelite.cmake)
 else()
   set(cuml_FOUND OFF)
 endif()
@@ -96,7 +96,7 @@ if(NOT ${CUML_CPU})
     set(CUML_EXCLUDE_TREELITE_FROM_ALL ON)
     set(CUML_EXCLUDE_CUMLPRIMS_MG_FROM_ALL ON)
 
-    add_subdirectory(../cpp cuml-cpp EXCLUDE_FROM_ALL)
+    add_subdirectory(${CUML_CPP_SRC} cuml-cpp EXCLUDE_FROM_ALL)
 
     set(cython_lib_dir cuml)
     install(TARGETS ${CUML_CPP_TARGET} DESTINATION ${cython_lib_dir})

python/cuml/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

python/cuml/cuml/VERSION

@@ -0,0 +1 @@
+../../../VERSION

python/cuml/common/CMakeLists.txt → python/cuml/cuml/common/CMakeLists.txt

@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -35,5 +35,5 @@ if(${CUML_UNIVERSAL})
 # todo: ml_cuda_utils.h should be in the include folder of cuML or the functionality
 # moved to another file, pointer_utils.pyx needs it
 # https://github.com/rapidsai/cuml/issues/4841
-  target_include_directories(common_pointer_utils PRIVATE "../../../cpp/src/")
+  target_include_directories(common_pointer_utils PRIVATE "../../../../cpp/src/")
 endif()