Remove cudf-polars[rapidsmpf] pip extra & numpy as a [test] dependency; add [dask] pip extra

dependencies.yaml

@@ -334,14 +334,23 @@ files:
       - depends_on_rapidsmpf
       - depends_on_pylibcudf
       - depends_on_cuda_python
+  # TODO: Eventually remove this alias in favor of dask
   py_run_cudf_polars_experimental:
     output: pyproject
     pyproject_dir: python/cudf_polars
     extras:
       table: project.optional-dependencies
       key: experimental
     includes:
-      - run_cudf_polars_experimental
+      - run_cudf_polars_dask
+  py_run_cudf_polars_dask:
+    output: pyproject
+    pyproject_dir: python/cudf_polars
+    extras:
+      table: project.optional-dependencies
+      key: dask
+    includes:
+      - run_cudf_polars_dask
   py_run_cudf_polars_ray:
     output: pyproject
     pyproject_dir: python/cudf_polars
@@ -359,7 +368,6 @@ files:
     includes:
       - cuda_version
       - depends_on_dask_cuda
-      - numpy_run
       - test_python_common
       - test_python_cudf_polars
       - cudf_polars_trace
@@ -823,12 +831,11 @@ dependencies:
           - packaging
           - polars>=1.35,<1.39
           - typing_extensions>=4.0.0
-  run_cudf_polars_experimental:
+  run_cudf_polars_dask:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
           - rapids-dask-dependency==26.6.*,>=0.0.0a0
-          - *nvidia_ml_py
   run_dask_cudf:
     common:
       - output_types: [conda, requirements, pyproject]

python/cudf_polars/pyproject.toml

@@ -43,7 +43,6 @@ classifiers = [
 [project.optional-dependencies]
 test = [
     "dask-cuda==26.6.*,>=0.0.0a0",
-    "numpy>=1.26,<3.0",
     "pytest",
     "pytest-cov",
     "pytest-httpserver",
@@ -53,19 +52,17 @@ test = [
     "zstandard",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 experimental = [
-    "nvidia-ml-py>=12",
     "rapids-dask-dependency==26.6.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 trace = [
     "structlog",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
-rapidsmpf = [
-    "pyarrow>=19.0.0,<24",
-    "rapidsmpf==26.6.*,>=0.0.0a0",
-] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 ray = [
     "ray>=2.55.1",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+dask = [
+    "rapids-dask-dependency==26.6.*,>=0.0.0a0",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
 Homepage = "https://github.com/rapidsai/cudf"

python/cudf_polars/tests/experimental/test_allgather.py

@@ -7,9 +7,10 @@
 
 import asyncio
 
-import numpy as np
 from rapidsmpf.streaming.cudf.table_chunk import TableChunk
 
+import polars as pl
+
 import pylibcudf as plc
 
 from cudf_polars.experimental.rapidsmpf.collectives.allgather import AllGatherManager
@@ -24,7 +25,13 @@ async def _test_allgather(engine) -> None:
 
     # Create simple test tables with different sizes
     tables = [
-        plc.Table([plc.Column.from_array(np.full(num_elements, i).astype(np.int32))])
+        plc.Table(
+            [
+                plc.Column.from_arrow(
+                    pl.Series([i] * num_elements, dtype=pl.Int32()), stream=stream
+                )
+            ]
+        )
         for i, num_elements in enumerate([100, 200, 300])
     ]
 

python/cudf_polars/tests/experimental/test_spilling.py

@@ -5,16 +5,18 @@
 
 from __future__ import annotations
 
+import random
 from typing import TYPE_CHECKING
 
-import numpy as np
 import pytest
 from rapidsmpf.memory.buffer import MemoryType
 from rapidsmpf.memory.pinned_memory_resource import is_pinned_memory_resources_supported
 from rapidsmpf.streaming.core.message import Message
 from rapidsmpf.streaming.core.spillable_messages import SpillableMessages
 from rapidsmpf.streaming.cudf.table_chunk import TableChunk
 
+import polars as pl
+
 import pylibcudf as plc
 
 from cudf_polars.experimental.rapidsmpf.frontend.options import StreamingOptions
@@ -31,8 +33,9 @@ def create_test_table(nbytes: int, stream: Stream) -> plc.Table:
     assert nbytes % 4 == 0, "nbytes must be divisible by 4 for float32"
     # Create a simple table with one column of random float32 data
     num_elements = nbytes // 4
-    data = np.random.random(num_elements).astype(np.float32)
-    return plc.Table([plc.Column.from_array(data, stream=stream)])
+    rng = random.Random(42)
+    pl_data = pl.Series([rng.random() for _ in range(num_elements)], dtype=pl.Float32())
+    return plc.Table([plc.Column.from_arrow(pl_data, stream=stream)])
 
 
 @pytest.mark.parametrize(