[carbondata] branch master updated: [CARBONDATA-3909] Fix Global sort data load failure issue with Decimal value as NULL

[akashrn5]

This is an automated email from the ASF dual-hosted git repository.

akashrn5 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new ddbe5cd  [CARBONDATA-3909] Fix Global sort data load failure issue 
with Decimal value as NULL
ddbe5cd is described below

commit ddbe5cde9092718d328328432e947b460810f149
Author: Indhumathi27 <indhumathi...@gmail.com>
AuthorDate: Thu Jul 16 21:36:54 2020 +0530

    [CARBONDATA-3909] Fix Global sort data load failure issue with Decimal 
value as NULL
    
    Why is this PR needed?
    Global sort Data load having decimal value as NULL fails with NPE
    
    What changes were proposed in this PR?
    Added null check, if actual data is null for decimal types and assigned 
data to NULL value
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    Yes
    
    This closes #3851
---
 .../scala/org/apache/carbondata/spark/util/CommonUtil.scala   |  7 ++++++-
 .../spark/testsuite/dataload/TestGlobalSortDataLoad.scala     | 11 +++++++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git 
a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
 
b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index dcbc9d2..3114db1 100644
--- 
a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ 
b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -931,7 +931,12 @@ object CommonUtil {
               DataTypes.STRING)
           }
         case d: DecimalType =>
-          data(i) = row.getDecimal(i, d.precision, d.scale).toJavaBigDecimal
+          val decimalValue = row.getDecimal(i, d.precision, d.scale)
+          if (null == decimalValue) {
+            data(i) = null
+          } else {
+            data(i) = decimalValue.toJavaBigDecimal
+          }
         case arrayType: ArrayType =>
           val result = convertSparkComplexTypeToCarbonObject(row.get(i, 
arrayType), arrayType)
           // convert carbon complex object to byte array
diff --git 
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
 
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index f1851f1..acd47e1 100644
--- 
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ 
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -20,6 +20,8 @@ package org.apache.carbondata.spark.testsuite.dataload
 import scala.collection.JavaConverters._
 import java.io.{File, FileWriter}
 
+import scala.collection.mutable
+
 import org.apache.commons.io.FileUtils
 import org.apache.hadoop.conf.Configuration
 
@@ -470,14 +472,15 @@ class TestGlobalSortDataLoad extends QueryTest with 
BeforeAndAfterEach with Befo
       sql("SELECT * FROM carbon_localsort_difftypes ORDER BY shortField"))
   }
 
+  // CARBONDATA-3909 fix data load failure with null value for decimal type
   test("test global sort with null values") {
     sql("drop table if exists source")
     sql("drop table if exists sink")
-    sql("create table source(a string, b int, c int, d int, e int, f int) 
stored as carbondata TBLPROPERTIES('bad_record_action'='force')")
-    sql("insert into source select 'k','k', 'k','k','k', 'k'")
-    sql("create table sink (a string, b string, c int, d bigint, e double, f 
char(5)) stored as carbondata TBLPROPERTIES('sort_scope'='global_sort', 
'sort_columns'='b,c,d,f')")
+    sql("create table source(a string, b int, c int, d int, e int, f int, dec 
decimal(3,2), arr array<string>, str struct<a:string>, map map<string, string>) 
stored as carbondata TBLPROPERTIES('bad_record_action'='force')")
+    sql("insert into source select 'k','k', 'k','k','k', 
'k',null,null,null,map('null','null')")
+    sql("create table sink (a string, b string, c int, d bigint, e double, f 
char(5),  dec decimal(3,2), arr array<string>, str struct<a:string>, map 
map<string, string>) stored as carbondata 
TBLPROPERTIES('sort_scope'='global_sort', 'sort_columns'='b,c,d,f')")
     sql("insert into sink select * from source")
-    checkAnswer(sql("select * from sink"), Row("k", null, null,null,null, 
null))
+    checkAnswer(sql("select * from sink"), Row("k", null, null,null,null, 
null, null, mutable.WrappedArray.make(Array(null)), Row(null), Map("null" -> 
"null")))
   }
 
   private def resetConf() {