This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 41eadda [CARBONDATA-3963]Fix hive timestamp data mismatch issue and
empty data during query issue
41eadda is described below
commit 41eadda6cc854ec13cd96822b0aa2b92e222e04d
Author: akashrn5 <[email protected]>
AuthorDate: Wed Aug 26 14:23:58 2020 +0530
[CARBONDATA-3963]Fix hive timestamp data mismatch issue and empty data
during query issue
Why is this PR needed?
1. when carbon table is queried through beeline, gives empty data.
2. the timestamp data given by carbon table is different than hive table.
Because during read we used to send the Timestamp value to
WritableTimestampObjectInspector's
create method, which basically was converting the value to EpochMillli,
which resulted in the
time value go different than the actual input data.
What changes were proposed in this PR?
1. Pass the carbon storage_handler into the table parameters
2. Convert the the long value to formatted string and directly form the
TimestampWritableV2,
by passing the Hive's Timestamp object which avoid EpochMilli and gives the
proper data as output.
This closes #3903
---
.../java/org/apache/carbondata/hive/WritableReadSupport.java | 12 ++++++++----
.../apache/carbondata/hive/CarbonHiveMetastoreListener.scala | 2 ++
.../test/java/org/apache/carbondata/hive/HiveTestUtils.java | 11 +++++++++--
3 files changed, 19 insertions(+), 6 deletions(-)
diff --git
a/integration/hive/src/main/java/org/apache/carbondata/hive/WritableReadSupport.java
b/integration/hive/src/main/java/org/apache/carbondata/hive/WritableReadSupport.java
index b0d87e7..a404b78 100644
---
a/integration/hive/src/main/java/org/apache/carbondata/hive/WritableReadSupport.java
+++
b/integration/hive/src/main/java/org/apache/carbondata/hive/WritableReadSupport.java
@@ -18,12 +18,13 @@
package org.apache.carbondata.hive;
import java.io.IOException;
-import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -35,7 +36,6 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.serde2.io.*;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import
org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
@@ -231,8 +231,12 @@ public class WritableReadSupport<T> implements
CarbonReadSupport<T> {
} else if (dataType == DataTypes.DATE) {
return new DateWritableV2((Integer) obj);
} else if (dataType == DataTypes.TIMESTAMP) {
- WritableTimestampObjectInspector ins = new
WritableTimestampObjectInspector();
- return ins.getPrimitiveWritableObject(ins.create(new Timestamp((long)
obj / 1000)));
+ SimpleDateFormat dateFormat =
+ new
SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ String formattedTime = dateFormat.format((long) obj / 1000);
+ org.apache.hadoop.hive.common.type.Timestamp t =
+ org.apache.hadoop.hive.common.type.Timestamp.valueOf(formattedTime);
+ return new TimestampWritableV2(t);
} else if (dataType == DataTypes.STRING) {
return new Text(obj.toString());
} else if (DataTypes.isArrayType(dataType)) {
diff --git
a/integration/hive/src/main/scala/org/apache/carbondata/hive/CarbonHiveMetastoreListener.scala
b/integration/hive/src/main/scala/org/apache/carbondata/hive/CarbonHiveMetastoreListener.scala
index 3f0f4f0..c8e268b 100644
---
a/integration/hive/src/main/scala/org/apache/carbondata/hive/CarbonHiveMetastoreListener.scala
+++
b/integration/hive/src/main/scala/org/apache/carbondata/hive/CarbonHiveMetastoreListener.scala
@@ -51,6 +51,8 @@ class CarbonHiveMetastoreListener(conf: Configuration)
extends MetaStorePreEvent
table.getSd.setCols(hiveSchema)
table.getSd.setInputFormat("org.apache.carbondata.hive.MapredCarbonInputFormat")
table.getSd.setOutputFormat("org.apache.carbondata.hive.MapredCarbonOutputFormat")
+ table.getParameters
+ .put("storage_handler",
"org.apache.carbondata.hive.CarbonStorageHandler")
val serdeInfo = table.getSd.getSerdeInfo
serdeInfo.setSerializationLib("org.apache.carbondata.hive.CarbonHiveSerDe")
val tablePath = serdeInfo.getParameters.get("tablePath")
diff --git
a/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
b/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
index 67c65a4..44c19b7 100644
---
a/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
+++
b/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
@@ -21,6 +21,9 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import org.apache.carbondata.hive.test.server.HiveEmbeddedServer2;
@@ -56,6 +59,8 @@ public abstract class HiveTestUtils {
public boolean checkAnswer(ResultSet actual, ResultSet expected) throws
SQLException {
Assert.assertEquals("Row Count Mismatch: ", expected.getFetchSize(),
actual.getFetchSize());
int rowCountExpected = 0;
+ List<String> expectedValuesList = new ArrayList<>();
+ List<String> actualValuesList = new ArrayList<>();
while (expected.next()) {
rowCountExpected ++;
if (!actual.next()) {
@@ -65,10 +70,12 @@ public abstract class HiveTestUtils {
Assert.assertTrue(numOfColumnsExpected > 0);
Assert.assertEquals(actual.getMetaData().getColumnCount(),
numOfColumnsExpected);
for (int i = 1; i <= numOfColumnsExpected; i++) {
- Assert.assertEquals(actual.getString(i), actual.getString(i));
+ expectedValuesList.add(expected.getString(i));
+ actualValuesList.add(actual.getString(i));
}
- System.out.println();
}
+ Collections.sort(expectedValuesList);Collections.sort(actualValuesList);
+ Assert.assertArrayEquals(expectedValuesList.toArray(),
actualValuesList.toArray());
Assert.assertTrue(rowCountExpected > 0);
return true;
}
