This is an automated email from the ASF dual-hosted git repository.
gengliangwang pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new ba1b3b0dbfde [SPARK-56615][SQL] Throw overflow errors for ceil and
floor
ba1b3b0dbfde is described below
commit ba1b3b0dbfde11374e622820547b2a77e20373c1
Author: ycli12 <[email protected]>
AuthorDate: Wed Jul 8 09:34:01 2026 -0700
[SPARK-56615][SQL] Throw overflow errors for ceil and floor
### What changes were proposed in this pull request?
This PR makes `CEIL` and `FLOOR` throw `ARITHMETIC_OVERFLOW` for
out-of-range `DOUBLE` inputs when ANSI mode is enabled. The non-ANSI path keeps
the existing JVM cast behavior.
The change covers both interpreted evaluation and code generation, and
updates the V2 expression conversion call sites after adding the new
`failOnError` parameter.
Closes #56615.
### Why are the changes needed?
In ANSI mode, Spark should report arithmetic overflow instead of silently
converting out-of-range `DOUBLE` results to saturated `Long` values. `CEIL` and
`FLOOR` were missing this overflow check.
### Does this PR introduce any user-facing change?
Yes. With ANSI mode enabled, `CEIL` and `FLOOR` now fail with
`ARITHMETIC_OVERFLOW` when the rounded `DOUBLE` result cannot be represented as
`Long`. With ANSI mode disabled, the previous behavior is preserved.
### How was this patch tested?
- `JAVA_HOME=/opt/homebrew/opt/openjdk17 LC_ALL=C ./build/sbt
"catalyst/clean" "catalyst/compile"`
- `JAVA_HOME=/opt/homebrew/opt/openjdk17 LC_ALL=C ./build/sbt
"catalyst/Test/testOnly
org.apache.spark.sql.catalyst.expressions.MathExpressionsSuite -- -z ceil"`
- `JAVA_HOME=/opt/homebrew/opt/openjdk17 LC_ALL=C ./build/sbt
"catalyst/Test/testOnly
org.apache.spark.sql.catalyst.expressions.MathExpressionsSuite -- -z floor"`
- `JAVA_HOME=/opt/homebrew/opt/openjdk17 LC_ALL=C ./build/sbt
"catalyst/Test/testOnly
org.apache.spark.sql.catalyst.expressions.MathExpressionsSuite"`
- `SPARK_GENERATE_GOLDEN_FILES=1 JAVA_HOME=/opt/homebrew/opt/openjdk17
LC_ALL=C ./build/sbt "sql/testOnly org.apache.spark.sql.SQLQueryTestSuite -- -z
postgreSQL/float8.sql"`
- `JAVA_HOME=/opt/homebrew/opt/openjdk17 LC_ALL=C ./build/sbt "sql/testOnly
org.apache.spark.sql.SQLQueryTestSuite -- -z postgreSQL/float8.sql"`
- `JAVA_HOME=/opt/homebrew/opt/openjdk17 LC_ALL=C ./build/sbt
-Phive-thriftserver "hive-thriftserver/testOnly
org.apache.spark.sql.hive.thriftserver.ThriftServerQueryTestSuite -- -z
postgreSQL/float8.sql"`
- `JAVA_HOME=/opt/homebrew/opt/openjdk17 LC_ALL=C ./build/sbt "sql/testOnly
org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite -- -z
math"`
- `git diff --check`
Closes #56921 from ycli12/codex/spark-ceil-floor-overflow.
Lead-authored-by: ycli12 <[email protected]>
Co-authored-by: yuechengli <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>
(cherry picked from commit c76b9e29a346b4c3866e229223e304667cda1f38)
Signed-off-by: Gengliang Wang <[email protected]>
---
.../catalyst/expressions/V2ExpressionUtils.scala | 4 +-
.../sql/catalyst/expressions/mathExpressions.scala | 64 +++++++++++++++++--
.../expressions/MathExpressionsSuite.scala | 54 +++++++++++-----
.../sql-tests/results/postgreSQL/float8.sql.out | 72 ++++++++++++++++------
4 files changed, 155 insertions(+), 39 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
index c561677ed5ad..702255e07574 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
@@ -315,8 +315,8 @@ object V2ExpressionUtils extends SQLConfHelper with Logging
{
case "EXP" => convertUnaryExpr(expr, Exp)
case "POWER" => convertBinaryExpr(expr, Pow)
case "SQRT" => convertUnaryExpr(expr, Sqrt)
- case "FLOOR" => convertUnaryExpr(expr, Floor)
- case "CEIL" => convertUnaryExpr(expr, Ceil)
+ case "FLOOR" => convertUnaryExpr(expr, Floor(_))
+ case "CEIL" => convertUnaryExpr(expr, Ceil(_))
case "ROUND" => convertBinaryExpr(expr, Round(_, _, ansiEnabled = true))
case "CBRT" => convertUnaryExpr(expr, Cbrt)
case "DEGREES" => convertUnaryExpr(expr, ToDegrees)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index 7bd166250db0..7a76968018c0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -252,7 +252,36 @@ case class Cbrt(child: Expression) extends
UnaryMathExpression(math.cbrt, "CBRT"
override protected def withNewChildInternal(newChild: Expression): Cbrt =
copy(child = newChild)
}
-case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil,
"CEIL") {
+private object CeilFloor {
+ def doubleToLong(value: Double, context: QueryContext): Long = {
+ if (!value.isNaN &&
+ (value < Long.MinValue.toDouble || value >= Long.MaxValue.toDouble)) {
+ throw QueryExecutionErrors.arithmeticOverflowError("long overflow",
context = context)
+ }
+ value.toLong
+ }
+
+ def doubleToLongCode(
+ input: String,
+ funcName: String,
+ result: String,
+ roundedValue: String,
+ errorContext: String): String = {
+ s"""
+ |double $roundedValue = java.lang.Math.$funcName($input);
+ |if (!java.lang.Double.isNaN($roundedValue) &&
+ | ($roundedValue < (double) java.lang.Long.MIN_VALUE ||
+ | $roundedValue >= (double) java.lang.Long.MAX_VALUE)) {
+ | throw QueryExecutionErrors.arithmeticOverflowError(
+ | "long overflow", "", $errorContext);
+ |}
+ |$result = (long) $roundedValue;
+ |""".stripMargin
+ }
+}
+
+case class Ceil(child: Expression, failOnError: Boolean =
SQLConf.get.ansiEnabled)
+ extends UnaryMathExpression(math.ceil, "CEIL") with SupportQueryContext {
override def dataType: DataType = child.dataType match {
case dt @ DecimalType.Fixed(_, 0) => dt
case DecimalType.Fixed(precision, scale) =>
@@ -260,11 +289,17 @@ case class Ceil(child: Expression) extends
UnaryMathExpression(math.ceil, "CEIL"
case _ => LongType
}
+ override def initQueryContext(): Option[QueryContext] = {
+ if (failOnError) Some(origin.context) else None
+ }
+
override def inputTypes: Seq[AbstractDataType] =
Seq(TypeCollection(DoubleType, DecimalType, LongType))
protected override def nullSafeEval(input: Any): Any = child.dataType match {
case LongType => input.asInstanceOf[Long]
+ case DoubleType if failOnError =>
+ CeilFloor.doubleToLong(f(input.asInstanceOf[Double]), getContextOrNull())
case DoubleType => f(input.asInstanceOf[Double]).toLong
case DecimalType.Fixed(_, _) => input.asInstanceOf[Decimal].ceil
}
@@ -275,6 +310,12 @@ case class Ceil(child: Expression) extends
UnaryMathExpression(math.ceil, "CEIL"
case DecimalType.Fixed(_, _) =>
defineCodeGen(ctx, ev, c => s"$c.ceil()")
case LongType => defineCodeGen(ctx, ev, c => s"$c")
+ case DoubleType if failOnError =>
+ nullSafeCodeGen(ctx, ev, c => {
+ val roundedValue = ctx.freshName("roundedValue")
+ val errorContext = getContextOrNullCode(ctx)
+ CeilFloor.doubleToLongCode(c, funcName, ev.value, roundedValue,
errorContext)
+ })
case _ => defineCodeGen(ctx, ev, c =>
s"(long)(java.lang.Math.${funcName}($c))")
}
}
@@ -531,7 +572,8 @@ case class Expm1(child: Expression) extends
UnaryMathExpression(StrictMath.expm1
override protected def withNewChildInternal(newChild: Expression): Expm1 =
copy(child = newChild)
}
-case class Floor(child: Expression) extends UnaryMathExpression(math.floor,
"FLOOR") {
+case class Floor(child: Expression, failOnError: Boolean =
SQLConf.get.ansiEnabled)
+ extends UnaryMathExpression(math.floor, "FLOOR") with SupportQueryContext {
override def dataType: DataType = child.dataType match {
case dt @ DecimalType.Fixed(_, 0) => dt
case DecimalType.Fixed(precision, scale) =>
@@ -539,11 +581,17 @@ case class Floor(child: Expression) extends
UnaryMathExpression(math.floor, "FLO
case _ => LongType
}
+ override def initQueryContext(): Option[QueryContext] = {
+ if (failOnError) Some(origin.context) else None
+ }
+
override def inputTypes: Seq[AbstractDataType] =
Seq(TypeCollection(DoubleType, DecimalType, LongType))
protected override def nullSafeEval(input: Any): Any = child.dataType match {
case LongType => input.asInstanceOf[Long]
+ case DoubleType if failOnError =>
+ CeilFloor.doubleToLong(f(input.asInstanceOf[Double]), getContextOrNull())
case DoubleType => f(input.asInstanceOf[Double]).toLong
case DecimalType.Fixed(_, _) => input.asInstanceOf[Decimal].floor
}
@@ -554,11 +602,17 @@ case class Floor(child: Expression) extends
UnaryMathExpression(math.floor, "FLO
case DecimalType.Fixed(_, _) =>
defineCodeGen(ctx, ev, c => s"$c.floor()")
case LongType => defineCodeGen(ctx, ev, c => s"$c")
+ case DoubleType if failOnError =>
+ nullSafeCodeGen(ctx, ev, c => {
+ val roundedValue = ctx.freshName("roundedValue")
+ val errorContext = getContextOrNullCode(ctx)
+ CeilFloor.doubleToLongCode(c, funcName, ev.value, roundedValue,
errorContext)
+ })
case _ => defineCodeGen(ctx, ev, c =>
s"(long)(java.lang.Math.${funcName}($c))")
}
- }
- override protected def withNewChildInternal(newChild: Expression): Floor =
- copy(child = newChild)
+ }
+ override protected def withNewChildInternal(newChild: Expression): Floor =
+ copy(child = newChild)
}
// scalastyle:off line.size.limit
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala
index c843c5640779..0ea3bc77ec1c 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala
@@ -393,13 +393,21 @@ class MathExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
test("ceil") {
- testUnary(Ceil, (d: Double) => math.ceil(d).toLong)
- checkConsistencyBetweenInterpretedAndCodegen(Ceil, DoubleType)
-
- testUnary(Ceil, (d: Decimal) => d.ceil, (-20 to 20).map(x => Decimal(x *
0.1)))
- checkConsistencyBetweenInterpretedAndCodegen(Ceil, DecimalType(25, 3))
- checkConsistencyBetweenInterpretedAndCodegen(Ceil, DecimalType(25, 0))
- checkConsistencyBetweenInterpretedAndCodegen(Ceil, DecimalType(5, 0))
+ testUnary(Ceil(_), (d: Double) => math.ceil(d).toLong)
+ checkConsistencyBetweenInterpretedAndCodegenAllowingException(
+ (child: Expression) => Ceil(child),
+ DoubleType)
+
+ testUnary(Ceil(_), (d: Decimal) => d.ceil, (-20 to 20).map(x => Decimal(x
* 0.1)))
+ checkConsistencyBetweenInterpretedAndCodegen(
+ (child: Expression) => Ceil(child),
+ DecimalType(25, 3))
+ checkConsistencyBetweenInterpretedAndCodegen(
+ (child: Expression) => Ceil(child),
+ DecimalType(25, 0))
+ checkConsistencyBetweenInterpretedAndCodegen(
+ (child: Expression) => Ceil(child),
+ DecimalType(5, 0))
val doublePi: Double = 3.1415
val floatPi: Float = 3.1415f
@@ -420,16 +428,29 @@ class MathExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(checkDataTypeAndCast(Ceil(1234567890123456L)),
1234567890123456L, EmptyRow)
checkEvaluation(checkDataTypeAndCast(Ceil(0.01)), 1L, EmptyRow)
checkEvaluation(checkDataTypeAndCast(Ceil(-0.10)), 0L, EmptyRow)
+
+ checkExceptionInExpression[SparkArithmeticException](
+ Ceil(Literal(1e30), failOnError = true),
+ "ARITHMETIC_OVERFLOW")
+ checkEvaluation(Ceil(Literal(1e30), failOnError = false), Long.MaxValue)
}
test("floor") {
- testUnary(Floor, (d: Double) => math.floor(d).toLong)
- checkConsistencyBetweenInterpretedAndCodegen(Floor, DoubleType)
-
- testUnary(Floor, (d: Decimal) => d.floor, (-20 to 20).map(x => Decimal(x *
0.1)))
- checkConsistencyBetweenInterpretedAndCodegen(Floor, DecimalType(25, 3))
- checkConsistencyBetweenInterpretedAndCodegen(Floor, DecimalType(25, 0))
- checkConsistencyBetweenInterpretedAndCodegen(Floor, DecimalType(5, 0))
+ testUnary(Floor(_), (d: Double) => math.floor(d).toLong)
+ checkConsistencyBetweenInterpretedAndCodegenAllowingException(
+ (child: Expression) => Floor(child),
+ DoubleType)
+
+ testUnary(Floor(_), (d: Decimal) => d.floor, (-20 to 20).map(x =>
Decimal(x * 0.1)))
+ checkConsistencyBetweenInterpretedAndCodegen(
+ (child: Expression) => Floor(child),
+ DecimalType(25, 3))
+ checkConsistencyBetweenInterpretedAndCodegen(
+ (child: Expression) => Floor(child),
+ DecimalType(25, 0))
+ checkConsistencyBetweenInterpretedAndCodegen(
+ (child: Expression) => Floor(child),
+ DecimalType(5, 0))
val doublePi: Double = 3.1415
val floatPi: Float = 3.1415f
@@ -450,6 +471,11 @@ class MathExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(checkDataTypeAndCast(Floor(1234567890123456L)),
1234567890123456L, EmptyRow)
checkEvaluation(checkDataTypeAndCast(Floor(0.01)), 0L, EmptyRow)
checkEvaluation(checkDataTypeAndCast(Floor(-0.10)), -1L, EmptyRow)
+
+ checkExceptionInExpression[SparkArithmeticException](
+ Floor(Literal(-1e30), failOnError = true),
+ "ARITHMETIC_OVERFLOW")
+ checkEvaluation(Floor(Literal(-1e30), failOnError = false), Long.MinValue)
}
test("factorial") {
diff --git
a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float8.sql.out
b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float8.sql.out
index a8f48f7cb4a9..689c2ba84045 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float8.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float8.sql.out
@@ -380,37 +380,73 @@ struct<five:string,f1:double,round_f1:double>
-- !query
select ceil(f1) as ceil_f1 from float8_tbl f
-- !query schema
-struct<ceil_f1:bigint>
+struct<>
-- !query output
--34
-0
-1
-1005
-9223372036854775807
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "ARITHMETIC_OVERFLOW",
+ "sqlState" : "22003",
+ "messageParameters" : {
+ "alternative" : "",
+ "config" : "\"spark.sql.ansi.enabled\"",
+ "message" : "overflow"
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 15,
+ "fragment" : "ceil(f1)"
+ } ]
+}
-- !query
select ceiling(f1) as ceiling_f1 from float8_tbl f
-- !query schema
-struct<ceiling_f1:bigint>
+struct<>
-- !query output
--34
-0
-1
-1005
-9223372036854775807
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "ARITHMETIC_OVERFLOW",
+ "sqlState" : "22003",
+ "messageParameters" : {
+ "alternative" : "",
+ "config" : "\"spark.sql.ansi.enabled\"",
+ "message" : "overflow"
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 18,
+ "fragment" : "ceiling(f1)"
+ } ]
+}
-- !query
select floor(f1) as floor_f1 from float8_tbl f
-- !query schema
-struct<floor_f1:bigint>
+struct<>
-- !query output
--35
-0
-0
-1004
-9223372036854775807
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "ARITHMETIC_OVERFLOW",
+ "sqlState" : "22003",
+ "messageParameters" : {
+ "alternative" : "",
+ "config" : "\"spark.sql.ansi.enabled\"",
+ "message" : "overflow"
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 16,
+ "fragment" : "floor(f1)"
+ } ]
+}
-- !query
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
