This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new aa4ef0e [SPARK-37196][SQL] HiveDecimal enforcePrecisionScale failed
return null
aa4ef0e is described below
commit aa4ef0e889184bb145c3aa3795deeaef87d5e819
Author: Angerszhuuuu <[email protected]>
AuthorDate: Mon Nov 8 12:08:32 2021 -0800
[SPARK-37196][SQL] HiveDecimal enforcePrecisionScale failed return null
For case
```
withTempDir { dir =>
withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") {
withTable("test_precision") {
val df = sql("SELECT 'dummy' AS name,
1000000000000000000010.7000000000000010 AS value")
df.write.mode("Overwrite").parquet(dir.getAbsolutePath)
sql(
s"""
|CREATE EXTERNAL TABLE test_precision(name STRING, value
DECIMAL(18,6))
|STORED AS PARQUET LOCATION '${dir.getAbsolutePath}'
|""".stripMargin)
checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy",
null))
}
}
}
```
We write a data with schema
It's caused by you create a df with
```
root
|-- name: string (nullable = false)
|-- value: decimal(38,16) (nullable = false)
```
but create table schema
```
root
|-- name: string (nullable = false)
|-- value: decimal(18,6) (nullable = false)
```
This will cause enforcePrecisionScale return `null`
```
public HiveDecimal getPrimitiveJavaObject(Object o) {
return o == null ? null :
this.enforcePrecisionScale(((HiveDecimalWritable)o).getHiveDecimal());
}
```
Then throw NPE when call `toCatalystDecimal `
We should judge if the return value is `null` to avoid throw NPE
Fix bug
No
Added UT
Closes #34519 from AngersZhuuuu/SPARK-37196.
Authored-by: Angerszhuuuu <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit a4f8ffbbfb0158a03ff52f1ed0dde75241c3a90e)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../main/scala/org/apache/spark/sql/hive/HiveShim.scala | 15 ++++++++++++---
.../apache/spark/sql/hive/execution/SQLQuerySuite.scala | 17 +++++++++++++++++
2 files changed, 29 insertions(+), 3 deletions(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
index 04a6a8f..e408ef9 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
@@ -103,10 +103,19 @@ private[hive] object HiveShim {
def toCatalystDecimal(hdoi: HiveDecimalObjectInspector, data: Any): Decimal
= {
if (hdoi.preferWritable()) {
-
Decimal(hdoi.getPrimitiveWritableObject(data).getHiveDecimal().bigDecimalValue,
- hdoi.precision(), hdoi.scale())
+ val value = hdoi.getPrimitiveWritableObject(data)
+ if (value == null) {
+ null
+ } else {
+ Decimal(value.getHiveDecimal().bigDecimalValue, hdoi.precision(),
hdoi.scale())
+ }
} else {
- Decimal(hdoi.getPrimitiveJavaObject(data).bigDecimalValue(),
hdoi.precision(), hdoi.scale())
+ val value = hdoi.getPrimitiveJavaObject(data)
+ if (value == null) {
+ null
+ } else {
+ Decimal(value.bigDecimalValue(), hdoi.precision(), hdoi.scale())
+ }
}
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index a46db32..3c0f902 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2558,6 +2558,23 @@ abstract class SQLQuerySuiteBase extends QueryTest with
SQLTestUtils with TestHi
}
}
}
+
+ test("SPARK-37196: HiveDecimal Precision Scale match failed should return
null") {
+ withTempDir { dir =>
+ withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") {
+ withTable("test_precision") {
+ val df = sql(s"SELECT 'dummy' AS name, ${"1" * 20}.${"2" * 18} AS
value")
+ df.write.mode("Overwrite").parquet(dir.getAbsolutePath)
+ sql(
+ s"""
+ |CREATE EXTERNAL TABLE test_precision(name STRING, value
DECIMAL(18,6))
+ |STORED AS PARQUET LOCATION '${dir.getAbsolutePath}'
+ |""".stripMargin)
+ checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy", null))
+ }
+ }
+ }
+ }
}
@SlowHiveTest
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
