Skip to content

[GLUTEN-12260][VL] Fix CheckOverflowTransformer using wrong child type for cast decision in Spark-33 #12261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Xtpacz wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[GLUTEN-12260][VL] Fix CheckOverflowTransformer using wrong child type for cast decision in Spark-33 #12261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e37b064
[VL] Fix CheckOverflowTransformer using wrong child type for cast dec…
Xtpacz Jun 8, 2026
375b9f2
rerun CI
Xtpacz Jun 8, 2026
a4d8a79
rerun CI
Xtpacz Jun 10, 2026
cb57f6c
add ut
Xtpacz Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...en-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ case class CheckOverflowTransformer(
context,
substraitExprName,
child.doTransform(context),
original.child.dataType,
child.dataType,

@ulysses-you ulysses-you Jun 15, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Xtpacz can we add a test case to ensure we can transform all Spark version's CheckOverflow?

@Xtpacz Xtpacz Jun 17, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for your review! Added targeted CheckOverflow transformation tests in spark33/34/35/40/41 ut modules.

@philo-he philo-he Jun 17, 2026 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Xtpacz, thanks for the update. How about adding the test to the gluten-ut/test module to reduce duplicate code if it's independent of any backend? A new test suite could be created for it. BTW, if some test code diverges across Spark versions, you might use testWithSpecifiedSparkVersion.

original.dataType,
original.nullable,
original.nullOnOverflow)
Expand Down
27 changes: 27 additions & 0 deletions ...rc/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.expressions

import org.apache.gluten.expression._
import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.utils.BackendTestUtils

import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute}
Expand Down Expand Up @@ -135,4 +137,29 @@ class GlutenDecimalPrecisionSuite extends GlutenTestsTrait {
checkType(Divide(expr, u), DoubleType)
}
}

test("CheckOverflow transformer casts transformed child type") {
if (BackendTestUtils.isVeloxBackendLoaded()) {
val targetType = DecimalType(38, 17)
val transformedChildType = DecimalType(38, 18)
val original = CheckOverflow(
Literal(Decimal(0, targetType.precision, targetType.scale), targetType),
targetType,
nullOnOverflow = true)
val child = LiteralTransformer(
Literal(
Decimal(0, transformedChildType.precision, transformedChildType.scale),
transformedChildType))
assert(original.child.dataType != child.dataType)

val transformedNode =
CheckOverflowTransformer(ExpressionNames.CHECK_OVERFLOW, child, original)
.doTransform(new SubstraitContext)
.toProtobuf
assert(transformedNode.hasCast)
val castType = transformedNode.getCast.getType.getDecimal
assert(castType.getPrecision == targetType.precision)
assert(castType.getScale == targetType.scale)
}
}
}
27 changes: 27 additions & 0 deletions ...rc/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.expressions

import org.apache.gluten.expression._
import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.utils.BackendTestUtils

import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute}
Expand Down Expand Up @@ -135,4 +137,29 @@ class GlutenDecimalPrecisionSuite extends GlutenTestsTrait {
checkType(Divide(expr, u), DoubleType)
}
}

test("CheckOverflow transformer casts transformed child type") {
if (BackendTestUtils.isVeloxBackendLoaded()) {
val targetType = DecimalType(38, 17)
val transformedChildType = DecimalType(38, 18)
val original = CheckOverflow(
Literal(Decimal(0, targetType.precision, targetType.scale), targetType),
targetType,
nullOnOverflow = true)
val child = LiteralTransformer(
Literal(
Decimal(0, transformedChildType.precision, transformedChildType.scale),
transformedChildType))
assert(original.child.dataType != child.dataType)

val transformedNode =
CheckOverflowTransformer(ExpressionNames.CHECK_OVERFLOW, child, original)
.doTransform(new SubstraitContext)
.toProtobuf
assert(transformedNode.hasCast)
val castType = transformedNode.getCast.getType.getDecimal
assert(castType.getPrecision == targetType.precision)
assert(castType.getScale == targetType.scale)
}
}
}
27 changes: 27 additions & 0 deletions ...rc/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.expressions

import org.apache.gluten.expression._
import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.utils.BackendTestUtils

import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute}
Expand Down Expand Up @@ -135,4 +137,29 @@ class GlutenDecimalPrecisionSuite extends GlutenTestsTrait {
checkType(Divide(expr, u), DoubleType)
}
}

test("CheckOverflow transformer casts transformed child type") {
if (BackendTestUtils.isVeloxBackendLoaded()) {
val targetType = DecimalType(38, 17)
val transformedChildType = DecimalType(38, 18)
val original = CheckOverflow(
Literal(Decimal(0, targetType.precision, targetType.scale), targetType),
targetType,
nullOnOverflow = true)
val child = LiteralTransformer(
Literal(
Decimal(0, transformedChildType.precision, transformedChildType.scale),
transformedChildType))
assert(original.child.dataType != child.dataType)

val transformedNode =
CheckOverflowTransformer(ExpressionNames.CHECK_OVERFLOW, child, original)
.doTransform(new SubstraitContext)
.toProtobuf
assert(transformedNode.hasCast)
val castType = transformedNode.getCast.getType.getDecimal
assert(castType.getPrecision == targetType.precision)
assert(castType.getScale == targetType.scale)
}
}
}
27 changes: 27 additions & 0 deletions ...rc/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.expressions

import org.apache.gluten.expression._
import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.utils.BackendTestUtils

import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute}
Expand Down Expand Up @@ -135,4 +137,29 @@ class GlutenDecimalPrecisionSuite extends GlutenTestsTrait {
checkType(Divide(expr, u), DoubleType)
}
}

test("CheckOverflow transformer casts transformed child type") {
if (BackendTestUtils.isVeloxBackendLoaded()) {
val targetType = DecimalType(38, 17)
val transformedChildType = DecimalType(38, 18)
val original = CheckOverflow(
Literal(Decimal(0, targetType.precision, targetType.scale), targetType),
targetType,
nullOnOverflow = true)
val child = LiteralTransformer(
Literal(
Decimal(0, transformedChildType.precision, transformedChildType.scale),
transformedChildType))
assert(original.child.dataType != child.dataType)

val transformedNode =
CheckOverflowTransformer(ExpressionNames.CHECK_OVERFLOW, child, original)
.doTransform(new SubstraitContext)
.toProtobuf
assert(transformedNode.hasCast)
val castType = transformedNode.getCast.getType.getDecimal
assert(castType.getPrecision == targetType.precision)
assert(castType.getScale == targetType.scale)
}
}
}
27 changes: 27 additions & 0 deletions ...rc/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.expressions

import org.apache.gluten.expression._
import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.utils.BackendTestUtils

import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute}
Expand Down Expand Up @@ -135,4 +137,29 @@ class GlutenDecimalPrecisionSuite extends GlutenTestsTrait {
checkType(Divide(expr, u), DoubleType)
}
}

test("CheckOverflow transformer casts transformed child type") {
if (BackendTestUtils.isVeloxBackendLoaded()) {
val targetType = DecimalType(38, 17)
val transformedChildType = DecimalType(38, 18)
val original = CheckOverflow(
Literal(Decimal(0, targetType.precision, targetType.scale), targetType),
targetType,
nullOnOverflow = true)
val child = LiteralTransformer(
Literal(
Decimal(0, transformedChildType.precision, transformedChildType.scale),
transformedChildType))
assert(original.child.dataType != child.dataType)

val transformedNode =
CheckOverflowTransformer(ExpressionNames.CHECK_OVERFLOW, child, original)
.doTransform(new SubstraitContext)
.toProtobuf
assert(transformedNode.hasCast)
val castType = transformedNode.getCast.getType.getDecimal
assert(castType.getPrecision == targetType.precision)
assert(castType.getScale == targetType.scale)
}
}
}
Loading