Update to use Rapids 24.10 at runtime and build setup (rapidsai#8)

The main change here is the build setup @jameslamb.

The `pylibcudf` changes were started by Matt before, the C change, is
just a very simple change in public API (there is the new stream
parameter).

*~Draft, since the update to 25.01.00.dev should happen first~ Ready for
review, but I suspect tests will fail due to a small (fixed!) legate
issue for another day*.

---------

Signed-off-by: Sebastian Berg <[email protected]>
Co-authored-by: Matthew Roeschke <[email protected]>

Author: seberg

README.md

@@ -121,11 +121,8 @@ if __name__ == "__main__":
 
 int main(int argc, char** argv)
 {
-  // First we initialize Legate and cuPyNumeric
-  int32_t errcode = legate::start(argc, argv);
-  if (errcode != 0) {
-    throw std::runtime_error("legate::start() errorcode: " + std::to_string(errcode));
-  }
+  // First we initialize Legate use either `legate` or `LEGATE_CONFIG` to customize launch
+  legate::start();
 
   // Then let's create a new logical column
   legate::dataframe::LogicalColumn col_a = legate::dataframe::sequence(20, -10);

build.sh

@@ -155,7 +155,7 @@ if (( NUMARGS == 0 )) || hasArg legate_dataframe; then
     cd ${REPODIR}/python/
     SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBLEGATE_DATAFRAME_BUILD_DIR} ${EXTRA_CMAKE_ARGS}" \
         SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL:-1}" \
-        python -m pip install --no-build-isolation --no-deps ${VERBOSE_FLAG} .
+        python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${VERBOSE_FLAG} .
 fi
 
 

conda/environments/all_cuda-124_arch-x86_64.yaml

@@ -17,27 +17,28 @@ dependencies:
 - cuda-profiler-api
 - cuda-sanitizer-api
 - cuda-version=12.4
-- cudf==24.08.*,>=0.0.0a0
+- cudf==24.10.*,>=0.0.0a0
 - cupy>=12.0.0
 - cupynumeric==25.01.*,>=0.0.0.dev0
 - cxx-compiler
-- cython>=3.0.0
-- dask-cuda==24.08.*
-- dask-cudf==24.08.*
+- cython>=3.0.3
+- dask-cuda==24.10.*
+- dask-cudf==24.10.*
 - gcc_linux-64=11.*
 - legate==25.01.*,>=0.0.0.dev0
-- libcudf==24.08.*,>=0.0.0a0
-- librmm==24.08.*,>=0.0.0a0
+- libcudf==24.10.*,>=0.0.0a0
+- librmm==24.10.*,>=0.0.0a0
 - make
 - myst-parser>=4.0
 - ninja
 - numpy >=1.23,<3.0.0a0
 - openssh
 - pydata-sphinx-theme>=0.16.0
+- pylibcudf==24.10.*,>=0.0.0a0
 - pytest>=7.0
 - python>=3.10,<3.13
-- scikit-build>=0.18.0
-- setuptools>=70.0
+- rapids-build-backend>=0.3.2,<0.4.0.dev0
+- scikit-build-core>=0.10.0
 - sphinx>=8.0
 - sysroot_linux-64==2.17
 - valgrind

conda/recipes/legate-dataframe/conda_build_config.yaml

@@ -23,4 +23,4 @@ legate_version:
   - "=25.01.*,>=0.0.0.dev0"
 
 rapids_version:
-  - =24.08.*
+  - =24.10.*

conda/recipes/legate-dataframe/meta.yaml

@@ -61,14 +61,15 @@ requirements:
     # is compatible with an existing cupynumeric version.
     # (may also stabilize not using debug/sanitizer builds)
     - cupynumeric
-    - cython>=3.0.0
     - python
     - pip
-    - scikit-build >=0.18.0
-    - setuptools >=70.0
+    - cython >=3.0.3
+    - rapids-build-backend >=0.3.0,<0.4.0.dev0
+    - scikit-build-core >=0.10.0
     - cudf {{ rapids_version }}  # pulls in pyarrow?
     - libcudf {{ rapids_version }}
     - librmm {{ rapids_version }}
+    - pylibcudf {{ rapids_version }}
   run:
     - python
     # cuda-version is used to constrain __cuda

cpp/cmake/fetch_rapids.cmake

@@ -12,7 +12,7 @@
 # the License.
 # =============================================================================
 if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/LEGATE_DATAFRAME_RAPIDS.cmake)
-  file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.08/RAPIDS.cmake
+  file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.10/RAPIDS.cmake
        ${CMAKE_CURRENT_BINARY_DIR}/LEGATE_DATAFRAME_RAPIDS.cmake
   )
 endif()

cpp/cmake/thirdparty/get_cudf.cmake

@@ -51,5 +51,5 @@ function(find_and_configure_cudf)
 endfunction()
 
 find_and_configure_cudf(
-  VERSION 24.08 GIT_REPO https://github.com/rapidsai/cudf.git GIT_TAG branch-24.08
+  VERSION 24.10 GIT_REPO https://github.com/rapidsai/cudf.git GIT_TAG branch-24.10
 )

cpp/examples/hello.cpp

@@ -24,13 +24,10 @@
 #include <legate_dataframe/parquet.hpp>
 #include <legate_dataframe/unaryop.hpp>
 
-int main(int argc, char** argv)
+int main(void)
 {
-  // First we initialize Legate
-  int32_t errcode = legate::start(argc, argv);
-  if (errcode != 0) {
-    throw std::runtime_error("legate::start() errorcode: " + std::to_string(errcode));
-  }
+  // First we initialize Legate use either `legate` or `LEGATE_CONFIG` to customize launch
+  legate::start();
 
   // Then let's create a new logical column
   legate::dataframe::LogicalColumn col_a = legate::dataframe::sequence(20, -10);

cpp/examples/third_party/third_party_hello.cpp

@@ -22,11 +22,8 @@
 
 int main(int argc, char** argv)
 {
-  // First we initialize Legate
-  int32_t errcode = legate::start(argc, argv);
-  if (errcode != 0) {
-    throw std::runtime_error("legate::start() errorcode: " + std::to_string(errcode));
-  }
+  // First we initialize Legate use either `legate` or `LEGATE_CONFIG` to customize launch
+  legate::start();
 
   // Then let's create a new logical column
   legate::dataframe::LogicalColumn col_a = legate::dataframe::sequence(10, 0);

cpp/tests/main.cpp

@@ -19,24 +19,13 @@
 
 class Environment : public ::testing::Environment {
  public:
-  Environment(int argc, char** argv) : argc_(argc), argv_(argv) {}
-
-  void SetUp() override
-  {
-    const char* argv[] = {"./test", "--gpus", "2"};  // TODO: make configurable
-    int argc           = 3;
-    EXPECT_EQ(legate::start(argc, (char**)argv), 0);
-  }
+  void SetUp() override { legate::start(); }
   void TearDown() override { EXPECT_EQ(legate::finish(), 0); }
-
- private:
-  int argc_;
-  char** argv_;
 };
 
 int main(int argc, char** argv)
 {
   ::testing::InitGoogleTest(&argc, argv);
-  ::testing::AddGlobalTestEnvironment(new Environment(argc, argv));
+  ::testing::AddGlobalTestEnvironment(new Environment());
   return RUN_ALL_TESTS();
 }

cpp/tests/test_join.cpp

@@ -69,6 +69,7 @@ template <std::pair<std::unique_ptr<rmm::device_uvector<cudf::size_type>>,
             cudf::table_view const& left_keys,
             cudf::table_view const& right_keys,
             cudf::null_equality compare_nulls,
+            rmm::cuda_stream_view stream,
             rmm::device_async_resource_ref mr),
           cudf::out_of_bounds_policy oob_policy = cudf::out_of_bounds_policy::DONT_CHECK>
 std::unique_ptr<cudf::table> join_and_gather(
@@ -77,21 +78,22 @@ std::unique_ptr<cudf::table> join_and_gather(
   std::vector<cudf::size_type> const& left_on,
   std::vector<cudf::size_type> const& right_on,
   cudf::null_equality compare_nulls,
+  rmm::cuda_stream_view stream      = cudf::get_default_stream(),
   rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource())
 {
   auto left_selected  = left_input.select(left_on);
   auto right_selected = right_input.select(right_on);
   auto const [left_join_indices, right_join_indices] =
-    join_impl(left_selected, right_selected, compare_nulls, mr);
+    join_impl(left_selected, right_selected, compare_nulls, stream, mr);
 
   auto left_indices_span  = cudf::device_span<cudf::size_type const>{*left_join_indices};
   auto right_indices_span = cudf::device_span<cudf::size_type const>{*right_join_indices};
 
   auto left_indices_col  = cudf::column_view{left_indices_span};
   auto right_indices_col = cudf::column_view{right_indices_span};
 
-  auto left_result  = cudf::gather(left_input, left_indices_col, oob_policy);
-  auto right_result = cudf::gather(right_input, right_indices_col, oob_policy);
+  auto left_result  = cudf::gather(left_input, left_indices_col, oob_policy, stream, mr);
+  auto right_result = cudf::gather(right_input, right_indices_col, oob_policy, stream, mr);
 
   return std::make_unique<cudf::table>(
     legate::dataframe::concat(left_result->release(), right_result->release()));

dependencies.yaml

@@ -7,6 +7,7 @@ files:
       arch: [x86_64]
     includes:
       - build
+      - rapids_build_skbuild
       - cuda
       - cuda_version
       - depends_on_cudf
@@ -34,6 +35,14 @@ files:
     pyproject_dir: python
     extras:
       table: build-system
+    includes:
+      - rapids_build_skbuild
+  py_rapids_build:
+    output: pyproject
+    pyproject_dir: python/
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
     includes:
       - build
       - depends_on_legate
@@ -68,10 +77,8 @@ dependencies:
       - output_types: [conda, requirements, pyproject]
         packages:
           - cmake>=3.26.4,!=3.30.0
-          - cython>=3.0.0
+          - cython>=3.0.3
           - ninja
-          - scikit-build>=0.18.0
-          - setuptools>=70.0
       - output_types: [conda]
         packages:
           - c-compiler
@@ -92,6 +99,18 @@ dependencies:
             packages:
               - cuda-nvcc
 
+  rapids_build_skbuild:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - rapids-build-backend>=0.3.2,<0.4.0.dev0
+      - output_types: conda
+        packages:
+          - scikit-build-core>=0.10.0
+      - output_types: [requirements, pyproject]
+        packages:
+          - scikit-build-core[pyproject]>=0.10.0
+
   cuda_version:
     specific:
       - output_types: conda
@@ -156,8 +175,8 @@ dependencies:
         packages:
           - cupynumeric==25.01.*,>=0.0.0.dev0
           - pytest>=7.0
-          - dask-cuda==24.08.*
-          - dask-cudf==24.08.*
+          - dask-cuda==24.10.*
+          - dask-cudf==24.10.*
       - output_types: conda
         packages:
           - cuda-sanitizer-api
@@ -168,15 +187,17 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - &cudf_unsuffixed cudf==24.08.*,>=0.0.0a0
+          - &cudf_unsuffixed cudf==24.10.*,>=0.0.0a0
+          - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0
     specific:
       - output_types: [requirements, pyproject]
         matrices:
           - matrix:
               cuda: "12.*"
               cuda_suffixed: "true"
             packages:
-              - cudf-cu12==24.08.*,>=0.0.0a0
+              - cudf-cu12==24.10.*,>=0.0.0a0
+              - pylibcudf-cu12==24.10.*,>=0.0.0a0
           - {matrix: null, packages: [*cudf_unsuffixed]}
 
   depends_on_cupy:
@@ -202,30 +223,30 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - &libcudf_unsuffixed libcudf==24.08.*,>=0.0.0a0
+          - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0
     specific:
       - output_types: [requirements, pyproject]
         matrices:
           - matrix:
               cuda: "12.*"
               cuda_suffixed: "true"
             packages:
-              - libcudf-cu12==24.08.*,>=0.0.0a0
+              - libcudf-cu12==24.10.*,>=0.0.0a0
           - {matrix: null, packages: [*libcudf_unsuffixed]}
 
   depends_on_librmm:
     common:
       - output_types: conda
         packages:
-          - &librmm_unsuffixed librmm==24.08.*,>=0.0.0a0
+          - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0
     specific:
       - output_types: [requirements, pyproject]
         matrices:
           - matrix:
               cuda: "12.*"
               cuda_suffixed: "true"
             packages:
-              - librmm-cu12==24.08.*,>=0.0.0a0
+              - librmm-cu12==24.10.*,>=0.0.0a0
           - matrix:
             packages:
               - *librmm_unsuffixed

python/legate_dataframe/lib/binaryop.pyx

@@ -6,7 +6,7 @@
 
 from libc.stdint cimport int32_t
 
-from cudf._lib.pylibcudf.types cimport data_type
+from pylibcudf.types cimport data_type
 
 from legate_dataframe.lib.core.column cimport LogicalColumn, cpp_LogicalColumn
 from legate_dataframe.lib.core.data_type cimport as_data_type

python/legate_dataframe/lib/core/column.pxd

@@ -8,8 +8,8 @@ from libcpp cimport bool
 from libcpp.memory cimport unique_ptr
 from libcpp.string cimport string
 
-from cudf._lib.pylibcudf.libcudf.column.column cimport column, column_view
-from cudf._lib.pylibcudf.types cimport data_type
+from pylibcudf.libcudf.column.column cimport column, column_view
+from pylibcudf.types cimport data_type
 
 from legate_dataframe.lib.core.legate_task cimport cpp_AutoTask
 from legate_dataframe.lib.core.logical_array cimport cpp_LogicalArray

python/legate_dataframe/lib/core/column.pyx

@@ -11,7 +11,7 @@ from libcpp.string cimport string
 from libcpp.utility cimport move
 
 from cudf._lib.column cimport Column as cudfColumn
-from cudf._lib.pylibcudf.libcudf.column.column cimport column
+from pylibcudf.libcudf.column.column cimport column
 
 from legate_dataframe.lib.core.legate_task cimport get_auto_task_handle
 from legate_dataframe.lib.core.logical_array cimport cpp_LogicalArray

python/legate_dataframe/lib/core/data_type.pxd

@@ -4,7 +4,7 @@
 # distutils: language = c++
 # cython: language_level=3
 
-from cudf._lib.pylibcudf.types cimport data_type as cpp_cudf_type
+from pylibcudf.types cimport data_type as cpp_cudf_type
 
 
 cdef cpp_cudf_type as_data_type(data_type_like)

python/legate_dataframe/lib/core/data_type.pyx

@@ -5,12 +5,12 @@
 # cython: language_level=3
 
 
-from cudf._lib.pylibcudf.types cimport DataType
-from cudf._lib.pylibcudf.types cimport data_type as cpp_cudf_type
 from cudf._lib.types cimport underlying_type_t_type_id
+from pylibcudf.types cimport DataType
+from pylibcudf.types cimport data_type as cpp_cudf_type
 
 import cudf
-from cudf._lib import pylibcudf
+import pylibcudf
 from cudf._lib.types import PYLIBCUDF_TO_SUPPORTED_NUMPY_TYPES, dtype_to_pylibcudf_type
 from numpy.typing import DTypeLike
 

python/legate_dataframe/lib/core/scalar.pxd

@@ -7,7 +7,7 @@
 from libcpp cimport bool
 from libcpp.memory cimport unique_ptr
 
-from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar as cudf_cpp_scalar
+from pylibcudf.libcudf.scalar.scalar cimport scalar as cudf_cpp_scalar
 
 
 cdef extern from "legate.h" nogil:

python/legate_dataframe/lib/groupby_aggregation.pyx

@@ -8,13 +8,13 @@
 from libcpp.string cimport string
 from libcpp.vector cimport vector
 
-from cudf._lib.pylibcudf.aggregation cimport aggregation as cudf_agg
+from pylibcudf.aggregation cimport aggregation as cudf_agg
 
 from legate_dataframe.lib.core.table cimport LogicalTable, cpp_LogicalTable
 
 from typing import Iterable, Tuple
 
-from cudf._lib.pylibcudf.libcudf.aggregation import Kind as AggregationKind
+from pylibcudf.libcudf.aggregation import Kind as AggregationKind
 
 from legate_dataframe.utils import _track_provenance
 

python/legate_dataframe/lib/timestamps.pyx

@@ -8,7 +8,7 @@
 from libc.stdint cimport int32_t
 from libcpp.string cimport string
 
-from cudf._lib.pylibcudf.types cimport data_type
+from pylibcudf.types cimport data_type
 
 from legate_dataframe.lib.core.column cimport LogicalColumn, cpp_LogicalColumn
 from legate_dataframe.lib.core.data_type cimport as_data_type

python/pyproject.toml

@@ -20,7 +20,7 @@ description = "Basic cudf operations on Legate"
 license = {text = "Apache 2.0"}
 requires-python = ">=3.10"
 dependencies = [
-    "cudf==24.08.*,>=0.0.0a0",
+    "cudf==24.10.*,>=0.0.0a0",
     "legate==25.01.*,>=0.0.0.dev0",
     "numpy >=1.23,<3.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
@@ -33,25 +33,44 @@ License = "https://github.com/rapidsai/legate-dataframe/blob/main/LICENSE"
 [project.optional-dependencies]
 test = [
     "cupynumeric==25.01.*,>=0.0.0.dev0",
-    "dask-cuda==24.08.*",
-    "dask-cudf==24.08.*",
+    "dask-cuda==24.10.*",
+    "dask-cudf==24.10.*",
     "pytest>=7.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [build-system]
+build-backend = "rapids_build_backend.build"
+requires = [
+    "rapids-build-backend>=0.3.2,<0.4.0.dev0",
+    "scikit-build-core[pyproject]>=0.10.0",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
+
+[tool.rapids-build-backend]
+build-backend = "scikit_build_core.build"
+dependencies-file = "../dependencies.yaml"
+matrix-entry = "cuda_suffixed=true"
 requires = [
     "cmake>=3.26.4,!=3.30.0",
-    "cython>=3.0.0",
+    "cython>=3.0.3",
     "legate==25.01.*,>=0.0.0.dev0",
-    "libcudf==24.08.*,>=0.0.0a0",
+    "libcudf==24.10.*,>=0.0.0a0",
     "ninja",
-    "scikit-build>=0.18.0",
-    "setuptools>=70.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
-build-backend = "setuptools.build_meta"
 
-[tool.setuptools.dynamic]
-version = {file = "legate_dataframe/VERSION"}
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "../VERSION"
+regex = "(?P<value>.*)"
+
+[tool.scikit-build]
+build-dir = "build/{wheel_tag}"
+cmake.build-type = "Release"
+cmake.version = "CMakeLists.txt"
+minimum-version = "build-system.requires"
+ninja.make-fallback = true
+sdist.exclude = ["*tests*"]
+sdist.reproducible = true
+wheel.packages = ["legate_dataframe"]
 
 [tool.black]
 line-length = 88

python/setup.py

@@ -86,11 +86,11 @@ def test_scalar_input(cudf_column, op, scalar):
 
     res = binary_operation(col, scalar, op, cudf_column.dtype)
     expect = cudf_binaryop(cudf_column, cudf_scalar, op_str, cudf_column.dtype)
-    assert_frame_equal(res, cudf.Series(expect))
+    assert_frame_equal(res, expect)
 
     res = binary_operation(scalar, col, op, cudf_column.dtype)
     expect = cudf_binaryop(cudf_scalar, cudf_column, op_str, cudf_column.dtype)
-    assert_frame_equal(res, cudf.Series(expect))
+    assert_frame_equal(res, expect)
 
     with pytest.raises(ValueError, match="both inputs cannot be scalars"):
         binary_operation(scalar, scalar, op, cudf_column.dtype)

python/tests/test_binaryop.py

@@ -15,8 +15,8 @@
 import cudf
 import cupy
 import legate.core.types as lg_type
-from cudf._lib.pylibcudf.unary import UnaryOperator
 from legate.core import get_legate_runtime
+from pylibcudf.unary import UnaryOperator
 
 from legate_dataframe import LogicalColumn
 from legate_dataframe.lib.unaryop import unary_operation

python/tests/test_tasks.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import cudf
-import cudf._lib.binaryop
 import pytest
 
 from legate_dataframe import LogicalColumn

python/tests/test_timestamps.py

@@ -15,7 +15,7 @@
 import cudf
 import cupy
 import pytest
-from cudf._lib.pylibcudf.unary import UnaryOperator
+from pylibcudf.unary import UnaryOperator
 
 from legate_dataframe import LogicalColumn
 from legate_dataframe.lib.unaryop import unary_operation