diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 4c583a4e9cf..d92d5a0f88b 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -30,7 +30,7 @@ rapids-pip-retry install \ -v \ --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ - "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental,ray]" \ + "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,dask,ray]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" diff --git a/dependencies.yaml b/dependencies.yaml index db219240487..a4fc6bcb68c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -334,6 +334,7 @@ files: - depends_on_rapidsmpf - depends_on_pylibcudf - depends_on_cuda_python + # TODO(26.10): Remove this alias in favor of dask py_run_cudf_polars_experimental: output: pyproject pyproject_dir: python/cudf_polars @@ -341,7 +342,15 @@ files: table: project.optional-dependencies key: experimental includes: - - run_cudf_polars_experimental + - run_cudf_polars_dask + py_run_cudf_polars_dask: + output: pyproject + pyproject_dir: python/cudf_polars + extras: + table: project.optional-dependencies + key: dask + includes: + - run_cudf_polars_dask py_run_cudf_polars_ray: output: pyproject pyproject_dir: python/cudf_polars @@ -359,7 +368,6 @@ files: includes: - cuda_version - depends_on_dask_cuda - - numpy_run - test_python_common - test_python_cudf_polars - cudf_polars_trace @@ -832,12 +840,11 @@ dependencies: - packaging - polars>=1.35,<1.39 - typing_extensions>=4.0.0 - run_cudf_polars_experimental: + run_cudf_polars_dask: common: - output_types: [conda, requirements, pyproject] packages: - rapids-dask-dependency==26.6.*,>=0.0.0a0 - - *nvidia_ml_py run_dask_cudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 7c21055d0b4..91867fb16eb 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -43,7 +43,6 @@ classifiers = [ [project.optional-dependencies] test = [ "dask-cuda==26.6.*,>=0.0.0a0", - "numpy>=1.26,<3.0", "pytest", "pytest-cov", "pytest-httpserver", @@ -53,19 +52,17 @@ test = [ "zstandard", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. experimental = [ - "nvidia-ml-py>=12", "rapids-dask-dependency==26.6.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. trace = [ "structlog", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -rapidsmpf = [ - "pyarrow>=19.0.0,<24", - "rapidsmpf==26.6.*,>=0.0.0a0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. ray = [ "ray>=2.55.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +dask = [ + "rapids-dask-dependency==26.6.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/cudf" diff --git a/python/cudf_polars/tests/experimental/test_allgather.py b/python/cudf_polars/tests/experimental/test_allgather.py index 52c353044eb..3a68b3f3988 100644 --- a/python/cudf_polars/tests/experimental/test_allgather.py +++ b/python/cudf_polars/tests/experimental/test_allgather.py @@ -7,9 +7,10 @@ import asyncio -import numpy as np from rapidsmpf.streaming.cudf.table_chunk import TableChunk +import polars as pl + import pylibcudf as plc from cudf_polars.experimental.rapidsmpf.collectives.allgather import AllGatherManager @@ -24,7 +25,13 @@ async def _test_allgather(engine) -> None: # Create simple test tables with different sizes tables = [ - plc.Table([plc.Column.from_array(np.full(num_elements, i).astype(np.int32))]) + plc.Table( + [ + plc.Column.from_arrow( + pl.Series([i] * num_elements, dtype=pl.Int32()), stream=stream + ) + ] + ) for i, num_elements in enumerate([100, 200, 300]) ] diff --git a/python/cudf_polars/tests/experimental/test_spilling.py b/python/cudf_polars/tests/experimental/test_spilling.py index 7f79b911038..ef2f555c7b2 100644 --- a/python/cudf_polars/tests/experimental/test_spilling.py +++ b/python/cudf_polars/tests/experimental/test_spilling.py @@ -5,9 +5,9 @@ from __future__ import annotations +import random from typing import TYPE_CHECKING -import numpy as np import pytest from rapidsmpf.memory.buffer import MemoryType from rapidsmpf.memory.pinned_memory_resource import is_pinned_memory_resources_supported @@ -15,6 +15,8 @@ from rapidsmpf.streaming.core.spillable_messages import SpillableMessages from rapidsmpf.streaming.cudf.table_chunk import TableChunk +import polars as pl + import pylibcudf as plc from cudf_polars.experimental.rapidsmpf.frontend.options import StreamingOptions @@ -31,8 +33,9 @@ def create_test_table(nbytes: int, stream: Stream) -> plc.Table: assert nbytes % 4 == 0, "nbytes must be divisible by 4 for float32" # Create a simple table with one column of random float32 data num_elements = nbytes // 4 - data = np.random.random(num_elements).astype(np.float32) - return plc.Table([plc.Column.from_array(data, stream=stream)]) + rng = random.Random(42) + pl_data = pl.Series([rng.random() for _ in range(num_elements)], dtype=pl.Float32()) + return plc.Table([plc.Column.from_arrow(pl_data, stream=stream)]) @pytest.mark.parametrize(