This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 1c78af7 [CARBONDATA-3981] Presto filter check on binary, byte and
float datatypes
1c78af7 is described below
commit 1c78af7acab957e9ae7cf8a42a5626fdd2a0eb53
Author: akkio-97 <[email protected]>
AuthorDate: Thu Sep 10 21:35:22 2020 +0530
[CARBONDATA-3981] Presto filter check on binary, byte and float datatypes
Why is this PR needed?
Due to the absence of binary, byte and float datatypes check, there were
either empty results or a problem during object serialisation in presto filter
queries.
What changes were proposed in this PR?
Above datatype conversions and checks has been added in
prestoFIlterUtil.java
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #3920
---
.../apache/carbondata/core/util/DataTypeUtil.java | 5 +++++
.../apache/carbondata/presto/PrestoFilterUtil.java | 10 +++++++++-
.../apache/carbondata/presto/PrestoFilterUtil.java | 10 +++++++++-
.../PrestoTestNonTransactionalTableFiles.scala | 20 +++++++++++++++++++-
4 files changed, 42 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 19139e0..32381e7 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -327,6 +327,11 @@ public final class DataTypeUtil {
return null;
}
return Float.parseFloat(data);
+ } else if (actualDataType == DataTypes.BYTE) {
+ if (data.isEmpty()) {
+ return null;
+ }
+ return Byte.parseByte(data);
} else if (actualDataType == DataTypes.DOUBLE) {
if (data.isEmpty()) {
return null;
diff --git
a/integration/presto/src/main/prestodb/org/apache/carbondata/presto/PrestoFilterUtil.java
b/integration/presto/src/main/prestodb/org/apache/carbondata/presto/PrestoFilterUtil.java
index 0b9a7c8..d1d9e13 100755
---
a/integration/presto/src/main/prestodb/org/apache/carbondata/presto/PrestoFilterUtil.java
+++
b/integration/presto/src/main/prestodb/org/apache/carbondata/presto/PrestoFilterUtil.java
@@ -78,16 +78,22 @@ public class PrestoFilterUtil {
HiveType colType = columnHandle.getHiveType();
if (colType.equals(HiveType.HIVE_BOOLEAN)) {
return DataTypes.BOOLEAN;
+ } else if (colType.equals(HiveType.HIVE_BINARY)) {
+ return DataTypes.BINARY;
} else if (colType.equals(HiveType.HIVE_SHORT)) {
return DataTypes.SHORT;
} else if (colType.equals(HiveType.HIVE_INT)) {
return DataTypes.INT;
+ } else if (colType.equals(HiveType.HIVE_FLOAT)) {
+ return DataTypes.FLOAT;
} else if (colType.equals(HiveType.HIVE_LONG)) {
return DataTypes.LONG;
} else if (colType.equals(HiveType.HIVE_DOUBLE)) {
return DataTypes.DOUBLE;
} else if (colType.equals(HiveType.HIVE_STRING)) {
return DataTypes.STRING;
+ } else if (colType.equals(HiveType.HIVE_BYTE)) {
+ return DataTypes.BYTE;
} else if (colType.equals(HiveType.HIVE_DATE)) {
return DataTypes.DATE;
} else if (colType.equals(HiveType.HIVE_TIMESTAMP)) {
@@ -282,9 +288,11 @@ public class PrestoFilterUtil {
private static Object convertDataByType(Object rawData, HiveType type) {
if (type.equals(HiveType.HIVE_INT) || type.equals(HiveType.HIVE_SHORT)) {
return Integer.valueOf(rawData.toString());
+ } else if (type.equals(HiveType.HIVE_FLOAT)) {
+ return Float.intBitsToFloat((int) ((Long) rawData).longValue());
} else if (type.equals(HiveType.HIVE_LONG)) {
return rawData;
- } else if (type.equals(HiveType.HIVE_STRING)) {
+ } else if (type.equals(HiveType.HIVE_STRING) ||
type.equals(HiveType.HIVE_BINARY)) {
if (rawData instanceof Slice) {
return ((Slice) rawData).toStringUtf8();
} else {
diff --git
a/integration/presto/src/main/prestosql/org/apache/carbondata/presto/PrestoFilterUtil.java
b/integration/presto/src/main/prestosql/org/apache/carbondata/presto/PrestoFilterUtil.java
index e5449f0..310b214 100755
---
a/integration/presto/src/main/prestosql/org/apache/carbondata/presto/PrestoFilterUtil.java
+++
b/integration/presto/src/main/prestosql/org/apache/carbondata/presto/PrestoFilterUtil.java
@@ -78,16 +78,22 @@ public class PrestoFilterUtil {
HiveType colType = columnHandle.getHiveType();
if (colType.equals(HiveType.HIVE_BOOLEAN)) {
return DataTypes.BOOLEAN;
+ } else if (colType.equals(HiveType.HIVE_BINARY)) {
+ return DataTypes.BINARY;
} else if (colType.equals(HiveType.HIVE_SHORT)) {
return DataTypes.SHORT;
} else if (colType.equals(HiveType.HIVE_INT)) {
return DataTypes.INT;
+ } else if (colType.equals(HiveType.HIVE_FLOAT)) {
+ return DataTypes.FLOAT;
} else if (colType.equals(HiveType.HIVE_LONG)) {
return DataTypes.LONG;
} else if (colType.equals(HiveType.HIVE_DOUBLE)) {
return DataTypes.DOUBLE;
} else if (colType.equals(HiveType.HIVE_STRING)) {
return DataTypes.STRING;
+ } else if (colType.equals(HiveType.HIVE_BYTE)) {
+ return DataTypes.BYTE;
} else if (colType.equals(HiveType.HIVE_DATE)) {
return DataTypes.DATE;
} else if (colType.equals(HiveType.HIVE_TIMESTAMP)) {
@@ -282,9 +288,11 @@ public class PrestoFilterUtil {
private static Object convertDataByType(Object rawData, HiveType type) {
if (type.equals(HiveType.HIVE_INT) || type.equals(HiveType.HIVE_SHORT)) {
return Integer.valueOf(rawData.toString());
+ } else if (type.equals(HiveType.HIVE_FLOAT)) {
+ return Float.intBitsToFloat((int) ((Long) rawData).longValue());
} else if (type.equals(HiveType.HIVE_LONG)) {
return rawData;
- } else if (type.equals(HiveType.HIVE_STRING)) {
+ } else if (type.equals(HiveType.HIVE_STRING) ||
type.equals(HiveType.HIVE_BINARY)) {
if (rawData instanceof Slice) {
return ((Slice) rawData).toStringUtf8();
} else {
diff --git
a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
index 611f33d..6bee1ed 100644
---
a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
+++
b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
@@ -223,6 +223,12 @@ class PrestoTestNonTransactionalTableFiles extends
FunSuiteLike with BeforeAndAf
String.valueOf(i.toFloat / 2)))
i += 1
}
+ writer
+ .write(Array[String]("true",
+ String.valueOf(i),
+ String.valueOf("abc"),
+ String.valueOf(i.toDouble / 2),
+ String.valueOf(i.toFloat / 2)))
writer.close()
} catch {
case ex: Throwable => throw new RuntimeException(ex)
@@ -271,6 +277,14 @@ class PrestoTestNonTransactionalTableFiles extends
FunSuiteLike with BeforeAndAf
.executeQuery("SELECT COUNT(*) AS RESULT FROM files ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 3))
assert(actualResult.equals(expectedResult))
+
+ val FloatFilterResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("SELECT * FROM files where salary > 0.0")
+ assert(FloatFilterResult.length == 2 )
+
+ val ByteFilterResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("SELECT * FROM files where id = 2")
+ assert(ByteFilterResult.length == 1 )
cleanTestData()
}
@@ -302,7 +316,7 @@ class PrestoTestNonTransactionalTableFiles extends
FunSuiteLike with BeforeAndAf
buildTestDataOtherDataType(3, null, writerPathBinary)
val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("select id from files1 ")
- assert(actualResult.size == 3)
+ assert(actualResult.size == 4)
// check the binary byte Array size, as original hex encoded image byte
array size is 118198
assert(actualResult.head("id").asInstanceOf[Array[Byte]].length == 118198)
// validate some initial bytes
@@ -310,6 +324,10 @@ class PrestoTestNonTransactionalTableFiles extends
FunSuiteLike with BeforeAndAf
assert(actualResult.head("id").asInstanceOf[Array[Byte]](1) == 57)
assert(actualResult.head("id").asInstanceOf[Array[Byte]](2) == 53)
assert(actualResult.head("id").asInstanceOf[Array[Byte]](3) == 48)
+
+ val binaryFilterResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("select id from files1 where id = cast('abc' as
varbinary)");
+ assert(binaryFilterResult.length == 1)
FileUtils.deleteDirectory(new File(writerPathBinary))
}
