Move Python ucxx code into python/ucxx (#252)

Also fix up the isort configuration to get more correct results. Contributes to rapidsai/build-planning#33 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Peter Andreas Entschev (https://github.com/pentschev) - James Lamb (https://github.com/jameslamb) - Vyas Ramasubramani (https://github.com/vyasr) URL: #252
rapidsai · Jul 30, 2024 · e9d966a · e9d966a
1 parent 8cdbcf0
commit e9d966a
Show file tree

Hide file tree

Showing 95 changed files with 156 additions and 147 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -5,16 +5,16 @@ cpp/                                @rapidsai/ucxx-cpp-codeowners
 python/                             @rapidsai/ucxx-python-codeowners
 
 #cmake code owners
-cpp/CMakeLists.txt                  @rapidsai/ucxx-cmake-codeowners
-cpp/python/CMakeLists.txt           @rapidsai/ucxx-cmake-codeowners
-cpp/examples/CMakeLists.txt         @rapidsai/ucxx-cmake-codeowners
-cpp/benchmarks/CMakeLists.txt       @rapidsai/ucxx-cmake-codeowners
-cpp/tests/CMakeLists.txt            @rapidsai/ucxx-cmake-codeowners
-python/CMakeLists.txt               @rapidsai/ucxx-cmake-codeowners
-python/ucxx/examples/CMakeLists.txt @rapidsai/ucxx-cmake-codeowners
-python/ucxx/_lib/CMakeLists.txt     @rapidsai/ucxx-cmake-codeowners
-fetch_rapids.cmake                  @rapidsai/ucxx-cmake-codeowners
-**/cmake/                           @rapidsai/ucxx-cmake-codeowners
+cpp/CMakeLists.txt                       @rapidsai/ucxx-cmake-codeowners
+cpp/python/CMakeLists.txt                @rapidsai/ucxx-cmake-codeowners
+cpp/examples/CMakeLists.txt              @rapidsai/ucxx-cmake-codeowners
+cpp/benchmarks/CMakeLists.txt            @rapidsai/ucxx-cmake-codeowners
+cpp/tests/CMakeLists.txt                 @rapidsai/ucxx-cmake-codeowners
+python/ucxx/CMakeLists.txt               @rapidsai/ucxx-cmake-codeowners
+python/ucxx/ucxx/examples/CMakeLists.txt @rapidsai/ucxx-cmake-codeowners
+python/ucxx/ucxx/_lib/CMakeLists.txt     @rapidsai/ucxx-cmake-codeowners
+fetch_rapids.cmake                       @rapidsai/ucxx-cmake-codeowners
+**/cmake/                                @rapidsai/ucxx-cmake-codeowners
 
 #CI code owners
 /.github/                @rapidsai/ci-codeowners

diff --git a/.gitignore b/.gitignore
@@ -16,11 +16,11 @@ dask-worker-space
 dist/
 python/distributed-ucxx/record.txt
 python/record.txt
-python/ucxx/_lib/*.a
-python/ucxx/_lib/*.o
-python/ucxx/_lib/*.c
-python/ucxx/_lib/*.cpp
-python/ucxx/_lib/*.h
+python/ucxx/ucxx/_lib/*.a
+python/ucxx/ucxx/_lib/*.o
+python/ucxx/ucxx/_lib/*.c
+python/ucxx/ucxx/_lib/*.cpp
+python/ucxx/ucxx/_lib/*.h
 wheels/
 *.egg-info/
 _skbuild/

diff --git a/build.sh b/build.sh
@@ -40,7 +40,7 @@ HELP="$0 [clean] [libucxx] [libucxx_python] [ucxx] [distributed_ucxx] [benchmark
    default action (no args) is to build and install 'libucxx' and 'libucxx_python', then 'ucxx' targets, and finally 'distributed_ucxx'
 "
 LIB_BUILD_DIR=${LIB_BUILD_DIR:=${REPODIR}/cpp/build}
-UCXX_BUILD_DIR=${REPODIR}/python/build
+UCXX_BUILD_DIR=${REPODIR}/python/ucxx/build
 
 BUILD_DIRS="${LIB_BUILD_DIR} ${UCXX_BUILD_DIR}"
 
@@ -224,7 +224,7 @@ if buildAll || hasArg ucxx; then
         export SKBUILD_INSTALL_STRIP=${SKBUILD_INSTALL_STRIP:-false}
     fi
 
-    cd ${REPODIR}/python/
+    cd ${REPODIR}/python/ucxx/
     SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_BUILD_TYPE=${BUILD_TYPE};${SKBUILD_EXTRA_CMAKE_ARGS}" \
         python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true .
 fi

diff --git a/build_and_run.sh b/build_and_run.sh
@@ -134,13 +134,13 @@ run_tests_async() {
   ENABLE_PYTHON_FUTURE=$3
   SKIP=$4
 
-  CMD_LINE="UCXPY_PROGRESS_MODE=${PROGRESS_MODE} UCXPY_ENABLE_DELAYED_SUBMISSION=${ENABLE_DELAYED_SUBMISSION} UCXPY_ENABLE_PYTHON_FUTURE=${ENABLE_PYTHON_FUTURE} pytest -vs python/ucxx/_lib_async/tests/"
+  CMD_LINE="UCXPY_PROGRESS_MODE=${PROGRESS_MODE} UCXPY_ENABLE_DELAYED_SUBMISSION=${ENABLE_DELAYED_SUBMISSION} UCXPY_ENABLE_PYTHON_FUTURE=${ENABLE_PYTHON_FUTURE} pytest -vs python/ucxx/ucxx/_lib_async/tests/"
 
   if [ $SKIP -ne 0 ]; then
     echo -e "\e[31;1mSkipping unstable test: ${CMD_LINE}\e[0m"
   else
     echo -e "\e[1mRunning: ${CMD_LINE}\e[0m"
-    UCXPY_PROGRESS_MODE=${PROGRESS_MODE} UCXPY_ENABLE_DELAYED_SUBMISSION=${ENABLE_DELAYED_SUBMISSION} UCXPY_ENABLE_PYTHON_FUTURE=${ENABLE_PYTHON_FUTURE} pytest -vs python/ucxx/_lib_async/tests/
+    UCXPY_PROGRESS_MODE=${PROGRESS_MODE} UCXPY_ENABLE_DELAYED_SUBMISSION=${ENABLE_DELAYED_SUBMISSION} UCXPY_ENABLE_PYTHON_FUTURE=${ENABLE_PYTHON_FUTURE} pytest -vs python/ucxx/ucxx/_lib_async/tests/
   fi
 }
 
@@ -190,8 +190,8 @@ if [[ $RUN_CPP_EXAMPLE != 0 ]]; then
   run_cpp_example   wait
 fi
 if [[ $RUN_PY_TESTS != 0 ]]; then
-  echo -e "\e[1mRunning: pytest-vs python/ucxx/_lib/tests/\e[0m"
-  pytest -vs python/ucxx/_lib/tests/
+  echo -e "\e[1mRunning: pytest-vs python/ucxx/ucxx/_lib/tests/\e[0m"
+  pytest -vs python/ucxx/ucxx/_lib/tests/
 fi
 if [[ $RUN_PY_ASYNC_TESTS != 0 ]]; then
   # run_tests_async PROGRESS_MODE   ENABLE_DELAYED_SUBMISSION ENABLE_PYTHON_FUTURE SKIP

diff --git a/ci/build_wheel_ucxx.sh b/ci/build_wheel_ucxx.sh
@@ -3,6 +3,6 @@
 
 set -euo pipefail
 
-package_dir="python"
+package_dir="python/ucxx"
 
 ./ci/build_wheel.sh ucxx ${package_dir}
diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -49,7 +49,7 @@ for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml; do
     sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
   done
-  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/pyproject.toml
+  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/ucxx/pyproject.toml
   sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/distributed-ucxx/pyproject.toml
 done
 
@@ -61,7 +61,7 @@ for DEP in "${UCXX_DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml; do
     sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/,>=0.0.0a0/g" "${FILE}"
   done
-  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/pyproject.toml
+  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/ucxx/pyproject.toml
   sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/distributed-ucxx/pyproject.toml
 done
 

diff --git a/ci/test_common.sh b/ci/test_common.sh
@@ -117,9 +117,9 @@ run_cpp_port_retry() {
 
 #################################### Python ####################################
 run_py_tests() {
-  CMD_LINE="timeout 2m python -m pytest -vs python/ucxx/_lib/tests/"
+  CMD_LINE="timeout 2m python -m pytest -vs python/ucxx/ucxx/_lib/tests/"
   log_command "${CMD_LINE}"
-  timeout 2m python -m pytest -vs python/ucxx/_lib/tests/
+  timeout 2m python -m pytest -vs python/ucxx/ucxx/_lib/tests/
 }
 
 run_py_tests_async() {
@@ -128,13 +128,13 @@ run_py_tests_async() {
   ENABLE_PYTHON_FUTURE=$3
   SKIP=$4
 
-  CMD_LINE="UCXPY_PROGRESS_MODE=${PROGRESS_MODE} UCXPY_ENABLE_DELAYED_SUBMISSION=${ENABLE_DELAYED_SUBMISSION} UCXPY_ENABLE_PYTHON_FUTURE=${ENABLE_PYTHON_FUTURE} timeout 20m python -m pytest -vs python/ucxx/_lib_async/tests/ --durations=50"
+  CMD_LINE="UCXPY_PROGRESS_MODE=${PROGRESS_MODE} UCXPY_ENABLE_DELAYED_SUBMISSION=${ENABLE_DELAYED_SUBMISSION} UCXPY_ENABLE_PYTHON_FUTURE=${ENABLE_PYTHON_FUTURE} timeout 20m python -m pytest -vs python/ucxx/ucxx/_lib_async/tests/ --durations=50"
 
   if [ $SKIP -ne 0 ]; then
     echo -e "\e[1;33mSkipping unstable test: ${CMD_LINE}\e[0m"
   else
     log_command "${CMD_LINE}"
-    UCXPY_PROGRESS_MODE=${PROGRESS_MODE} UCXPY_ENABLE_DELAYED_SUBMISSION=${ENABLE_DELAYED_SUBMISSION} UCXPY_ENABLE_PYTHON_FUTURE=${ENABLE_PYTHON_FUTURE} timeout 20m python -m pytest -vs python/ucxx/_lib_async/tests/ --durations=50
+    UCXPY_PROGRESS_MODE=${PROGRESS_MODE} UCXPY_ENABLE_DELAYED_SUBMISSION=${ENABLE_DELAYED_SUBMISSION} UCXPY_ENABLE_PYTHON_FUTURE=${ENABLE_PYTHON_FUTURE} timeout 20m python -m pytest -vs python/ucxx/ucxx/_lib_async/tests/ --durations=50
   fi
 }
 

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -58,14 +58,14 @@ files:
       - py_version
   py_rapids_build_ucxx:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/ucxx
     extras:
       table: build-system
     includes:
       - rapids_build_skbuild
   py_build_ucxx:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/ucxx
     extras:
       table: tool.rapids-build-backend
       key: requires
@@ -75,7 +75,7 @@ files:
       - depends_on_ucx_build
   py_run_ucxx:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/ucxx
     extras:
       table: project
     includes:
@@ -84,7 +84,7 @@ files:
       - depends_on_ucx_run
   py_test_ucxx:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/ucxx
     extras:
       table: project.optional-dependencies
       key: test

diff --git a/python/LICENSE b/python/LICENSE
diff --git a/python/README.md b/python/README.md
diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -1,56 +1,3 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
-# SPDX-License-Identifier: BSD-3-Clause
-
-[build-system]
-build-backend = "rapids_build_backend.build"
-requires = [
-    "rapids-build-backend>=0.3.0,<0.4.0.dev0",
-    "scikit-build-core[pyproject]>=0.7.0",
-] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
-
-[project]
-name = "ucxx"
-dynamic = ["version"]
-description = "Python Bindings for the Unified Communication X library (UCX)"
-readme = { file = "README.md", content-type = "text/markdown" }
-authors = [
-    { name = "NVIDIA Corporation" },
-]
-license = { text = "BSD-3-Clause" }
-requires-python = ">=3.9"
-dependencies = [
-    "libucx>=1.15.0",
-    "numpy>=1.23,<2.0a0",
-    "pynvml>=11.4.1",
-    "rmm==24.10.*,>=0.0.0a0",
-] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
-classifiers = [
-    "Intended Audience :: Developers",
-    "Intended Audience :: System Administrators",
-    "License :: OSI Approved :: BSD License",
-    "Operating System :: POSIX :: Linux",
-    "Programming Language :: Python",
-    "Topic :: Software Development :: Libraries :: Python Modules",
-    "Topic :: System :: Hardware",
-    "Topic :: System :: Systems Administration",
-    "Programming Language :: Python :: 3",
-]
-
-[project.optional-dependencies]
-test = [
-    "cloudpickle",
-    "cudf==24.10.*,>=0.0.0a0",
-    "cupy-cuda11x>=12.0.0",
-    "numba>=0.57.1",
-    "pytest-asyncio",
-    "pytest-rerunfailures",
-    "pytest==7.*",
-    "rapids-dask-dependency==24.10.*,>=0.0.0a0",
-] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
-
-[project.urls]
-Homepage = "https://github.com/rapidsai/ucxx"
-
 [tool.isort]
 line_length = 79
 multi_line_output = 3
@@ -67,7 +14,8 @@ known_rapids = [
     "cudf",
 ]
 known_first_party = [
-    "ucp",
+    "ucxx",
+    "distributed_ucxx",
 ]
 default_section = "THIRDPARTY"
 sections = [
@@ -91,32 +39,6 @@ skip = [
     "__init__.py",
 ]
 
-[tool.rapids-build-backend]
-build-backend = "scikit_build_core.build"
-dependencies-file = "../dependencies.yaml"
-matrix-entry = "cuda_suffixed=true"
-requires = [
-    "cmake>=3.26.4,!=3.30.0",
-    "cython>=3.0.0",
-    "libucx==1.15.0",
-    "ninja",
-    "rmm==24.10.*,>=0.0.0a0",
-] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
-
-[tool.scikit-build]
-build-dir = "build/{wheel_tag}"
-cmake.build-type = "Release"
-cmake.minimum-version = "3.26.4"
-ninja.make-fallback = true
-sdist.exclude = ["*tests*"]
-sdist.reproducible = true
-wheel.packages = ["ucxx"]
-
-[tool.scikit-build.metadata.version]
-provider = "scikit_build_core.metadata.regex"
-input = "ucxx/VERSION"
-regex = "(?P<value>.*)"
-
 [tool.ruff]
 select = ["E", "F", "W"]
 ignore = [

diff --git a/python/CMakeLists.txt → python/ucxx/CMakeLists.txt b/python/CMakeLists.txt → python/ucxx/CMakeLists.txt
@@ -5,15 +5,15 @@
 
 cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
 
-file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _version_contents)
+file(READ "${CMAKE_CURRENT_LIST_DIR}/../../VERSION" _version_contents)
 if(_version_contents MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*$")
   set(ucxx_version "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}")
 else()
   string(REPLACE "\n" "\n  " _version_contents_formatted "${_version_contents}")
   message(FATAL_ERROR "Could not determine ucxx version. Contents of VERSION file:\n  ${_version_contents_formatted}")
 endif()
 
-include(../fetch_rapids.cmake)
+include(../../fetch_rapids.cmake)
 
 project(
   ucxx-python
@@ -42,7 +42,7 @@ if(NOT ucxx_FOUND)
 
   set(_exclude_from_all "")
 
-  add_subdirectory(../cpp ucxx-cpp ${_exclude_from_all})
+  add_subdirectory(../../cpp ucxx-cpp ${_exclude_from_all})
 
   # Since ucxx._lib requires access to libucxx, we place the library in the ucxx directory
   # and modify the rpaths appropriately.

diff --git a/python/ucxx/LICENSE b/python/ucxx/LICENSE
@@ -0,0 +1 @@
+../../LICENSE
diff --git a/python/ucxx/README.md b/python/ucxx/README.md
@@ -0,0 +1 @@
+../../README.md
diff --git a/python/ucxx/VERSION b/python/ucxx/VERSION
diff --git a/python/examples/basic.py → python/ucxx/examples/basic.py b/python/examples/basic.py → python/ucxx/examples/basic.py
@@ -5,6 +5,7 @@
 import asyncio
 
 import numpy as np
+
 import ucxx._lib.libucxx as ucx_api
 from ucxx._lib.arr import Array
 from ucxx._lib_async.utils import get_event_loop