From 66ce2d09ea60091abcf3483979defa162eaf8877 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 21:56:53 +0000 Subject: [PATCH 1/3] Remove cudf_polars rapidsmpf job and consolidate into cudf_polars job --- .github/workflows/pr.yaml | 21 +------ .github/workflows/test.yaml | 21 +------ ci/run_cudf_polars_experimental_pytests.sh | 14 ----- ci/run_cudf_polars_pytests.sh | 2 +- ci/test_cudf_polars_experimental.sh | 69 ---------------------- ci/test_wheel_cudf_polars.sh | 2 +- 6 files changed, 5 insertions(+), 124 deletions(-) delete mode 100755 ci/run_cudf_polars_experimental_pytests.sh delete mode 100755 ci/test_cudf_polars_experimental.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f4e257d7764..e5447ffa95a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -339,7 +339,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false - ignored_pr_jobs: "telemetry-summarize spark-rapids-jni wheel-tests-cudf-polars-with-rapidsmpf" + ignored_pr_jobs: "telemetry-summarize spark-rapids-jni" conda-cpp-build: needs: checks permissions: @@ -605,25 +605,8 @@ jobs: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request - script: "ci/test_wheel_cudf_polars.sh" - wheel-tests-cudf-polars-with-rapidsmpf: - needs: [wheel-build-cudf-polars, changed-files] - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_nor_dask_cudf - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA" to minimize CI usage. - # (rapidsmpf compatibility already validated in rapidsmpf CI) - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" - script: "ci/test_cudf_polars_experimental.sh" + script: "ci/test_wheel_cudf_polars.sh" cudf-polars-polars-tests: needs: [wheel-build-cudf-polars, changed-files] permissions: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d75d4f58967..79d1c4d0423 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -246,31 +246,12 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main with: - build_type: ${{ inputs.build_type }} - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: "ci/test_wheel_cudf_polars.sh" - wheel-tests-cudf-polars-with-rapidsmpf: - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA" to minimize CI usage. - # (rapidsmpf compatibility already validated in rapidsmpf CI) - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" date: ${{ inputs.date }} sha: ${{ inputs.sha }} - script: "ci/test_cudf_polars_experimental.sh" - continue-on-error: true + script: "ci/test_wheel_cudf_polars.sh" cudf-polars-polars-tests: permissions: actions: read diff --git a/ci/run_cudf_polars_experimental_pytests.sh b/ci/run_cudf_polars_experimental_pytests.sh deleted file mode 100755 index da659c7b386..00000000000 --- a/ci/run_cudf_polars_experimental_pytests.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -# Test cudf_polars experimental. - -# It is essential to cd into python/cudf_polars as `pytest-xdist` + `coverage` seem to work only at this directory level. -# Support invoking outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_polars/ - -echo "Running the full cudf-polars test suite" -python -m pytest --cache-clear "$@" tests diff --git a/ci/run_cudf_polars_pytests.sh b/ci/run_cudf_polars_pytests.sh index 96e77c4b038..82d1ccd4879 100755 --- a/ci/run_cudf_polars_pytests.sh +++ b/ci/run_cudf_polars_pytests.sh @@ -8,4 +8,4 @@ set -euo pipefail # Support invoking run_cudf_polars_pytests.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_polars/ -python -m pytest --cache-clear "$@" tests --ignore=tests/experimental +python -m pytest --cache-clear "$@" tests diff --git a/ci/test_cudf_polars_experimental.sh b/ci/test_cudf_polars_experimental.sh deleted file mode 100755 index 4b796ff4b94..00000000000 --- a/ci/test_cudf_polars_experimental.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -source rapids-init-pip - -rapids-logger "Download wheels" - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" -CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-github python) -LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) -PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") - -rapids-logger "Installing cudf_polars and its dependencies (including rapidsmpf)" - -# generate constraints (possibly pinning to oldest support versions of dependencies) -rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" - -# notes: -# -# * echo to expand wildcard before adding `[test]` requires for pip -# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because -# that environment variable is ignored if any other --constraint are passed via the CLI -# -rapids-pip-retry install \ - -v \ - --prefer-binary \ - --constraint "${PIP_CONSTRAINT}" \ - "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental,ray]" \ - "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ - "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" - -rapids-logger "Run cudf_polars tests with rapidsmpf" - -# Get the latest polars version for testing -available_polars_versions=$(python -m pip index versions polars --json | jq '.versions') -POLARS_VERSION=$(python ci/utils/filter_package_versions.py dependencies.yaml run_cudf_polars polars "$available_polars_versions" | awk '{print $NF}') - -rapids-logger "Installing polars==${POLARS_VERSION}" -rapids-pip-retry install -U "polars==${POLARS_VERSION}" - -# shellcheck disable=SC2317 -function set_exitcode() -{ - EXITCODE=$? -} -EXITCODE=0 -trap set_exitcode ERR -set +e - -rapids-logger "Running cudf_polars experimental tests (non-ci-blocking)" -timeout 30m ./ci/run_cudf_polars_experimental_pytests.sh \ - --no-cov \ - --numprocesses=8 \ - --dist=worksteal \ - -v \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-rapidsmpf.xml" - -trap - ERR -set -e - -if [ ${EXITCODE} != 0 ]; then - rapids-logger "cudf_polars + rapidsmpf tests FAILED: exitcode ${EXITCODE}" -else - rapids-logger "cudf_polars + rapidsmpf tests PASSED" -fi -exit ${EXITCODE} diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 1ae3544e642..22a415836bb 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -30,7 +30,7 @@ rapids-pip-retry install \ -v \ --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ - "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental]" \ + "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental,ray]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" From 9c9d95e04248f6ad2b0d8fd3224ec97dd04dec1c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 12 May 2026 00:36:34 +0000 Subject: [PATCH 2/3] Change test_rolling_datetime to use a filterwarning --- python/cudf_polars/tests/experimental/test_rolling.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/python/cudf_polars/tests/experimental/test_rolling.py b/python/cudf_polars/tests/experimental/test_rolling.py index ee3ae137e27..a72d2d68244 100644 --- a/python/cudf_polars/tests/experimental/test_rolling.py +++ b/python/cudf_polars/tests/experimental/test_rolling.py @@ -20,6 +20,10 @@ def engine(streaming_engine_factory): ) +# HStack may redirect to Select before fallback; message differs by Polars IR / version. +@pytest.mark.filterwarnings( + "ignore:This .*is not supported for multiple partitions:UserWarning" +) def test_rolling_datetime(request, engine): if not POLARS_VERSION_LT_136: request.applymarker( @@ -39,12 +43,7 @@ def test_rolling_datetime(request, engine): .lazy() ) q = df.with_columns(pl.sum("a").rolling(index_column="dt", period="2d")) - # HStack may redirect to Select before fallback; message differs by Polars IR / version. - with pytest.warns( - UserWarning, - match=r"This (HStack|selection) is not supported for multiple partitions\.", - ): - assert_gpu_result_equal(q, engine=engine) + assert_gpu_result_equal(q, engine=engine) def test_over_in_filter_unsupported(request, streaming_engine_factory) -> None: From d3e03aac9b1ba0d1bface7523c3d0e1f9bb51ca3 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 12 May 2026 17:06:01 +0000 Subject: [PATCH 3/3] Use warns_on_spmd instead --- .../cudf_polars/tests/experimental/test_rolling.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/cudf_polars/tests/experimental/test_rolling.py b/python/cudf_polars/tests/experimental/test_rolling.py index a72d2d68244..bc1bbbc40eb 100644 --- a/python/cudf_polars/tests/experimental/test_rolling.py +++ b/python/cudf_polars/tests/experimental/test_rolling.py @@ -10,6 +10,7 @@ from cudf_polars.experimental.rapidsmpf.frontend.options import StreamingOptions from cudf_polars.experimental.rapidsmpf.frontend.spmd import SPMDEngine from cudf_polars.testing.asserts import assert_gpu_result_equal +from cudf_polars.testing.engine_utils import warns_on_spmd from cudf_polars.utils.versions import POLARS_VERSION_LT_136 @@ -20,10 +21,6 @@ def engine(streaming_engine_factory): ) -# HStack may redirect to Select before fallback; message differs by Polars IR / version. -@pytest.mark.filterwarnings( - "ignore:This .*is not supported for multiple partitions:UserWarning" -) def test_rolling_datetime(request, engine): if not POLARS_VERSION_LT_136: request.applymarker( @@ -43,7 +40,13 @@ def test_rolling_datetime(request, engine): .lazy() ) q = df.with_columns(pl.sum("a").rolling(index_column="dt", period="2d")) - assert_gpu_result_equal(q, engine=engine) + # HStack may redirect to Select before fallback; message differs by Polars IR / version. + with warns_on_spmd( + engine, + UserWarning, + match=r"This (HStack|selection) is not supported for multiple partitions\.", + ): + assert_gpu_result_equal(q, engine=engine) def test_over_in_filter_unsupported(request, streaming_engine_factory) -> None: