From 3edfa3760de6a1630fec868c360b6a2bd7e523c1 Mon Sep 17 00:00:00 2001 From: adityaanikam Date: Fri, 31 Jul 2026 20:48:55 +0530 Subject: [PATCH 1/5] Fix index misalignment in char_wb ngram padding --- python/cuml/cuml/feature_extraction/_vectorizers.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/cuml/cuml/feature_extraction/_vectorizers.py b/python/cuml/cuml/feature_extraction/_vectorizers.py index 716b9fe631..272869ab90 100644 --- a/python/cuml/cuml/feature_extraction/_vectorizers.py +++ b/python/cuml/cuml/feature_extraction/_vectorizers.py @@ -213,11 +213,15 @@ def get_char_ngrams(self, ngram_size, str_series, doc_id_sr): tokens = str_series.str.tokenize(self.delimiter) del str_series - padding = Series(self.delimiter).repeat(len(tokens)) + # tokens keeps the original per-document index (repeated per + # token); reset both to a plain range first so the two str.cat() + # calls below align positionally instead of by that index. + tokens = tokens.reset_index(drop=True) + padding = Series(self.delimiter).repeat(len(tokens)).reset_index( + drop=True + ) tokens = tokens.str.cat(padding) - padding = padding.reset_index(drop=True) tokens = padding.str.cat(tokens) - tokens = tokens.reset_index(drop=True) ngram_sr = tokens.str.character_ngrams(n=ngram_size) From ed13db728f6fd6dabceeacdb5cba0ecc7e5ccb7b Mon Sep 17 00:00:00 2001 From: adityaanikam Date: Wed, 19 Aug 2026 14:30:41 +0530 Subject: [PATCH 2/5] Add regression test for char_wb ngram index alignment --- .../cuml/tests/test_text_feature_extraction.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/cuml/tests/test_text_feature_extraction.py b/python/cuml/tests/test_text_feature_extraction.py index 8f6400bbd8..681c94c005 100644 --- a/python/cuml/tests/test_text_feature_extraction.py +++ b/python/cuml/tests/test_text_feature_extraction.py @@ -423,6 +423,22 @@ def test_tfidf_vectorizer_get_feature_names(): assert vectorizer.get_feature_names().to_arrow().to_pylist() == output +def test_tfidf_vectorizer_char_wb_ngrams(): + # Regression test for #8416: get_char_ngrams misaligned padded tokens + # across documents once index alignment relied on the original + # per-document index instead of a reset range index. + vectorizer = TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 6)) + tfidf_mat = vectorizer.fit_transform(DOCS_GPU) + + ref_vectorizer = SkTfidfVect(analyzer="char_wb", ngram_range=(2, 6)) + ref = ref_vectorizer.fit_transform(DOCS) + + cp.testing.assert_array_almost_equal(tfidf_mat.todense(), ref.toarray()) + assert vectorizer.get_feature_names().to_arrow().to_pylist() == list( + ref_vectorizer.get_feature_names_out() + ) + + # ---------------------------------------------------------------- # HashingVectorizer tests # ---------------------------------------------------------------- From 13943607c3493f242fa3b7d3e986a04581c4dae4 Mon Sep 17 00:00:00 2001 From: adityaanikam Date: Wed, 19 Aug 2026 14:43:10 +0530 Subject: [PATCH 3/5] Update char_wb ngram test for get_feature_names_out API change --- python/cuml/tests/test_text_feature_extraction.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/cuml/tests/test_text_feature_extraction.py b/python/cuml/tests/test_text_feature_extraction.py index 89fc37fdad..d3bd3c4d72 100644 --- a/python/cuml/tests/test_text_feature_extraction.py +++ b/python/cuml/tests/test_text_feature_extraction.py @@ -443,8 +443,9 @@ def test_tfidf_vectorizer_char_wb_ngrams(): ref = ref_vectorizer.fit_transform(DOCS) cp.testing.assert_array_almost_equal(tfidf_mat.todense(), ref.toarray()) - assert vectorizer.get_feature_names().to_arrow().to_pylist() == list( - ref_vectorizer.get_feature_names_out() + assert_array_equal( + vectorizer.get_feature_names_out(), + ref_vectorizer.get_feature_names_out(), ) From 213646af8cadbcb4d3076b0468a64c29f3f42061 Mon Sep 17 00:00:00 2001 From: Aditya Nikam <118399608+adityaanikam@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:59:40 +0530 Subject: [PATCH 4/5] Refactor padding assignment for better readability --- python/cuml/cuml/feature_extraction/_vectorizers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/cuml/cuml/feature_extraction/_vectorizers.py b/python/cuml/cuml/feature_extraction/_vectorizers.py index 5ec3747ddd..77ad35ddbf 100644 --- a/python/cuml/cuml/feature_extraction/_vectorizers.py +++ b/python/cuml/cuml/feature_extraction/_vectorizers.py @@ -218,8 +218,10 @@ def get_char_ngrams(self, ngram_size, str_series, doc_id_sr): # token); reset both to a plain range first so the two str.cat() # calls below align positionally instead of by that index. tokens = tokens.reset_index(drop=True) - padding = Series(self.delimiter).repeat(len(tokens)).reset_index( - drop=True + padding = ( + Series(self.delimiter) + .repeat(len(tokens)) + .reset_index(drop=True) ) tokens = tokens.str.cat(padding) tokens = padding.str.cat(tokens) From 9eb9f3f88f2f7dabe9f0019dd542bcb464a44aa9 Mon Sep 17 00:00:00 2001 From: adityaanikam Date: Sun, 23 Aug 2026 20:02:40 +0530 Subject: [PATCH 5/5] =?UTF-8?q?=EF=BB=BFFix=20ngram=5Fcount=20missing=20em?= =?UTF-8?q?pty-token=20documents=20in=20char=5Fwb=20branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI on this PR failed test_tfidf_vectorizer_char_wb_ngrams with KeyError: boolean label can not be used without a boolean index, in get_ngrams' ngram_count[not_empty_docs] filter. That test's DOCS fixture includes two empty-string documents, and this is the first char_wb test to include one. get_char_ngrams' char_wb branch computes ngram_count via doc_id_df.groupby("doc_id", sort=True).sum(). A document that tokenizes to zero tokens never contributes a row to doc_id_df, so its doc_id is entirely absent from the groupby result's index, rather than present with a count of 0. token_count, computed separately via str_series.str.token_count(), still has an entry for every document. The two indices then disagree, and get_ngrams' later not_empty_docs boolean mask, built from token_count's full index, no longer aligns with ngram_count's shorter one. Reindexed ngram_count onto token_count's index with fill_value=0 right after the groupby, restoring the same index for both that the other two branches (char, ngram_size == 1) already get for free since their ngram_count is arithmetic directly on token_count. This bug predates this PR and is unrelated to the padding order change in get_char_ngrams; it was only ever latent because no earlier char_wb test included a document with zero tokens. The regression test already added in this PR (test_tfidf_vectorizer_char_wb_ngrams) exercises it directly, so no additional test is needed. Could not run this locally, cuml requires an NVIDIA GPU and CUDA runtime; verified the fix by tracing the exact index mismatch against the failing job's log and reasoning through cudf.Series.reindex semantics, and am relying on CI to confirm. --- python/cuml/cuml/feature_extraction/_vectorizers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/cuml/cuml/feature_extraction/_vectorizers.py b/python/cuml/cuml/feature_extraction/_vectorizers.py index 77ad35ddbf..50b26f705a 100644 --- a/python/cuml/cuml/feature_extraction/_vectorizers.py +++ b/python/cuml/cuml/feature_extraction/_vectorizers.py @@ -241,6 +241,13 @@ def get_char_ngrams(self, ngram_size, str_series, doc_id_sr): ngram_count = doc_id_df.groupby("doc_id", sort=True).sum()[ "ngram_count" ] + # A document that tokenizes to zero tokens (e.g. an empty string) + # never appears in the groupby above, so its doc_id is silently + # missing from ngram_count's index instead of being present with + # a count of 0. Reindex onto token_count's full per-document + # index so the two stay aligned for get_ngrams' later + # ngram_count[not_empty_docs] boolean filter. + ngram_count = ngram_count.reindex(token_count.index, fill_value=0) return ngram_sr, ngram_count, token_count if ngram_size == 1: