This is an automated email from the ASF dual-hosted git repository.
indhumuthumurugesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 18665cc [CARBONDATA-4214] inserting NULL value when timestamp value
received from FROM_UNIXTIME(0)
18665cc is described below
commit 18665cc0b72d455ff66bfa63501faec94025b492
Author: Mahesh Raju Somalaraju <[email protected]>
AuthorDate: Fri Jun 18 00:40:18 2021 +0530
[CARBONDATA-4214] inserting NULL value when timestamp value received from
FROM_UNIXTIME(0)
Why is this PR needed?
Filling null in case of timestamp value is received from FROM_UNIXTIME(0)
as spark original
insert rdd value[internalRow] received in this case zero. if the original
column
value[internalRow] is zero then in insert flow adding NULL and giving NULL
to spark.
When query happens on the same column received NULL value instead of
timestamp value.
Problem code: if (internalRow.getLong(index) == 0) {
internalRow.setNullAt(index) }
What changes were proposed in this PR?
Removed the null filling check for zero value case and if internalRow value
is non
null/empty then only set the internalRow timestamp value.
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #4154
---
.../command/management/CommonLoadUtils.scala | 5 ++--
.../allqueries/AllDataTypesTestCase.scala | 29 ++++++++++++++++++++++
2 files changed, 31 insertions(+), 3 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
index 695681a..f2a8280 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
@@ -781,9 +781,8 @@ object CommonLoadUtils {
internalRowOriginal
}
for (index <- timeStampIndex) {
- if (internalRow.getLong(index) == 0) {
- internalRow.setNullAt(index)
- } else {
+ // timestmap value can be set other than null/empty case
+ if (!internalRow.isNullAt(index)) {
internalRow.setLong(
index,
internalRow.getLong(index) /
TimeStampGranularityTypeValue.MILLIS_SECONDS.getValue)
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
index 7044729..b95755c 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
@@ -17,6 +17,8 @@
package org.apache.carbondata.spark.testsuite.allqueries
+import java.sql.Timestamp
+
import org.apache.spark.sql.{Row, SaveMode}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
@@ -73,6 +75,33 @@ class AllDataTypesTestCase extends QueryTest with
BeforeAndAfterAll {
}
}
+ test("insert data with from_unixtime(0) and query") {
+ sql("drop table if exists time_carbon1")
+ sql("drop table if exists time_parquet")
+ sql("create table if not exists time_carbon1(time1 timestamp) stored as
carbondata")
+ sql("create table if not exists time_parquet(time1 timestamp) stored as
parquet")
+ sql("insert into time_carbon1 select from_unixtime(0)")
+ sql("insert into time_parquet select from_unixtime(0)")
+ sql("insert into time_carbon1 select from_unixtime(123456)")
+ sql("insert into time_parquet select from_unixtime(123456)")
+ checkAnswer(sql("select * from time_carbon1"),
+ sql("select * from time_parquet"))
+ sql("drop table if exists time_carbon1")
+ sql("drop table if exists time_parquet")
+ }
+
+ test("insert data with empty/null and query") {
+ sql("drop table if exists time_carbon2")
+ sql("create table if not exists time_carbon2(time1 timestamp) stored as
carbondata")
+ sql("insert into time_carbon2 select null")
+ checkAnswer(sql("select count(*) from time_carbon2"), Seq(Row(1)))
+ checkAnswer(sql("select * from time_carbon2"), Seq(Row(null)))
+ sql("insert into time_carbon2 select ''")
+ checkAnswer(sql("select count(*) from time_carbon2"), Seq(Row(2)))
+ checkAnswer(sql("select * from time_carbon2"), Seq(Row(null), Row(null)))
+ sql("drop table if exists time_carbon2")
+ }
+
// Test-24
test("select channelsId, sum(channelsId+ 10) Total from
Carbon_automation_test group by channelsId order by Total") {
checkAnswer(
