This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 8422f71 [CARBONDATA-3975]Fix wrong data from carbondata for binary
column when read via hive
8422f71 is described below
commit 8422f71c4c37c6693a6031edd599939cd27297aa
Author: akashrn5 <[email protected]>
AuthorDate: Fri Aug 28 10:55:18 2020 +0530
[CARBONDATA-3975]Fix wrong data from carbondata for binary column when read
via hive
Why is this PR needed?
When binary data read via hive, the results are not the actual
inserted data. This is because the BytesWritable objected will be
given for binary data, we were doing the toString() to get the data
object which gives wrong data for binary as it gives objects to string.
What changes were proposed in this PR?
For BytesWritable, get the bytes from the object and get the bytes
only for the specific length in object and not the extra bits.
This closes #3915
---
.../apache/carbondata/hive/CarbonHiveSerDe.java | 9 +++++++++
.../hive/src/main/resources/text/string.txt | 8 ++++++++
.../org/apache/carbondata/hive/HiveCarbonTest.java | 22 ++++++++++++++++++++++
.../org/apache/carbondata/hive/HiveTestUtils.java | 2 --
4 files changed, 39 insertions(+), 2 deletions(-)
diff --git
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveSerDe.java
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveSerDe.java
index b955655..f040c11 100644
---
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveSerDe.java
+++
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveSerDe.java
@@ -28,6 +28,7 @@ import javax.annotation.Nullable;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.SchemaReader;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
@@ -54,6 +55,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Writable;
import org.apache.log4j.Logger;
@@ -214,6 +216,13 @@ public class CarbonHiveSerDe extends AbstractSerDe {
if (obj == null) {
return null;
}
+ if
(inspector.getTypeInfo().getTypeName().equalsIgnoreCase(DataTypes.BINARY.getName()))
{
+ BytesWritable primitiveWritableObject =
+ (BytesWritable) inspector.getPrimitiveWritableObject(obj);
+ byte[] bytes = primitiveWritableObject.getBytes();
+ int length = primitiveWritableObject.getLength();
+ return Arrays.copyOfRange(bytes, 0, length);
+ }
return inspector.getPrimitiveWritableObject(obj).toString();
}
diff --git a/integration/hive/src/main/resources/text/string.txt
b/integration/hive/src/main/resources/text/string.txt
new file mode 100644
index 0000000..cfa91ed
--- /dev/null
+++ b/integration/hive/src/main/resources/text/string.txt
@@ -0,0 +1,8 @@
+abc 1
+test 2
+test 3
+ 8
+ 9
+testtest 4
+testtest 5
+testtest 6
diff --git
a/integration/hive/src/test/java/org/apache/carbondata/hive/HiveCarbonTest.java
b/integration/hive/src/test/java/org/apache/carbondata/hive/HiveCarbonTest.java
index b495933..6a11c86 100644
---
a/integration/hive/src/test/java/org/apache/carbondata/hive/HiveCarbonTest.java
+++
b/integration/hive/src/test/java/org/apache/carbondata/hive/HiveCarbonTest.java
@@ -42,6 +42,9 @@ public class HiveCarbonTest extends HiveTestUtils {
// "/complex" subdirectory name
private static final String COMPLEX = "complex";
+ // "/text" subdirectory name
+ private static final String TEXT = "text";
+
@BeforeClass
public static void setup() throws Exception {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT,
"false");
@@ -211,6 +214,25 @@ public class HiveCarbonTest extends HiveTestUtils {
checkAnswer(carbonResult, hiveResult);
}
+ @Test
+ public void testBinaryDataTypeColumns() throws Exception {
+ String dataPath = resourceDirectoryPath + TEXT + "/string.txt";
+ statement.execute("drop table if exists mytable_n2");
+ statement.execute("drop table if exists hive_carbon_table9");
+ statement.execute("drop table if exists mytable_n21");
+ statement.execute("CREATE TABLE mytable_n2(key binary, value int) ROW
FORMAT DELIMITED FIELDS TERMINATED BY '9'");
+ statement.execute("LOAD DATA LOCAL INPATH '" + dataPath + "' INTO TABLE
mytable_n2");
+ statement.execute("CREATE TABLE mytable_n21(key binary, value int) ROW
FORMAT DELIMITED FIELDS TERMINATED BY '9'");
+ statement.execute("insert into mytable_n21 select * from mytable_n2");
+ statement.execute(
+ "CREATE TABLE hive_carbon_table9(key binary, value int) "
+ + "stored by 'org.apache.carbondata.hive.CarbonStorageHandler'");
+ statement.execute("insert into hive_carbon_table9 select * from
mytable_n2");
+ ResultSet hiveResult = connection.createStatement().executeQuery("select *
from mytable_n2");
+ ResultSet carbonResult = connection.createStatement().executeQuery("select
* from hive_carbon_table9");
+ checkAnswer(carbonResult, hiveResult);
+ }
+
@AfterClass
public static void tearDown() {
try {
diff --git
a/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
b/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
index 951ac06..6eabc25 100644
---
a/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
+++
b/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
@@ -30,8 +30,6 @@ import
org.apache.carbondata.hive.test.server.HiveEmbeddedServer2;
import org.junit.Assert;
-import javax.validation.constraints.AssertTrue;
-
/**
* A utility class to start and stop the Hive Embedded Server.
*/
