This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 1890348 [SPARK-37451][SQL] Fix cast string type to decimal type if
spark.sql.legacy.allowNegativeScaleOfDecimal is enabled
1890348 is described below
commit 1890348daa4e4a46a79851ab826aed71b525e0bd
Author: Yuming Wang <[email protected]>
AuthorDate: Thu Dec 9 01:08:37 2021 -0800
[SPARK-37451][SQL] Fix cast string type to decimal type if
spark.sql.legacy.allowNegativeScaleOfDecimal is enabled
### What changes were proposed in this pull request?
Fix cast string type to decimal type only if
`spark.sql.legacy.allowNegativeScaleOfDecimal` is enabled. For example:
```scala
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
val data = Seq(Row("7.836725755512218E38"))
val schema = StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
```
The result is null since
[SPARK-32706](https://issues.apache.org/jira/browse/SPARK-32706).
### Why are the changes needed?
Fix regression bug.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit test.
Closes #34811 from wangyum/SPARK-37451.
Authored-by: Yuming Wang <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit a1214a98f4b3f7715fb984ad3df514471b1e33c7)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../main/scala/org/apache/spark/sql/types/Decimal.scala | 6 ++++--
.../scala/org/apache/spark/sql/types/DecimalSuite.scala | 15 +++++++++++++++
2 files changed, 19 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 4b1462b..cb468c5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -602,7 +602,8 @@ object Decimal {
val bigDecimal = stringToJavaBigDecimal(str)
// We fast fail because constructing a very large JavaBigDecimal to
Decimal is very slow.
// For example: Decimal("6.0790316E+25569151")
- if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION) {
+ if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION &&
+ !SQLConf.get.allowNegativeScaleOfDecimalEnabled) {
null
} else {
Decimal(bigDecimal)
@@ -618,7 +619,8 @@ object Decimal {
val bigDecimal = stringToJavaBigDecimal(str)
// We fast fail because constructing a very large JavaBigDecimal to
Decimal is very slow.
// For example: Decimal("6.0790316E+25569151")
- if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION) {
+ if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION &&
+ !SQLConf.get.allowNegativeScaleOfDecimalEnabled) {
throw QueryExecutionErrors.outOfDecimalTypeRangeError(str)
} else {
Decimal(bigDecimal)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 57449f5..5433c56 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -299,4 +299,19 @@ class DecimalSuite extends SparkFunSuite with
PrivateMethodTester with SQLHelper
assert(Decimal.fromStringANSI(UTF8String.fromString(string)) ===
Decimal(string))
}
}
+
+ test("SPARK-37451: Performance improvement regressed String to Decimal
cast") {
+ val values = Array("7.836725755512218E38")
+ for (string <- values) {
+ assert(Decimal.fromString(UTF8String.fromString(string)) === null)
+
intercept[ArithmeticException](Decimal.fromStringANSI(UTF8String.fromString(string)))
+ }
+
+ withSQLConf(SQLConf.LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED.key ->
"true") {
+ for (string <- values) {
+ assert(Decimal.fromString(UTF8String.fromString(string)) ===
Decimal(string))
+ assert(Decimal.fromStringANSI(UTF8String.fromString(string)) ===
Decimal(string))
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
