This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 54f8697 [CARBONDATA-4104] Vector filling for complex decimal type
needs to be handled
54f8697 is described below
commit 54f869715843d54f4aa5816266dcccd242cb5451
Author: akkio-97 <[email protected]>
AuthorDate: Fri Jan 8 01:21:59 2021 +0530
[CARBONDATA-4104] Vector filling for complex decimal type needs to be
handled
Why is this PR needed?
Filling of vectors in case of complex decimal type whose precision is
greater than 18 is not handled properly.
for ex-
array<decimal(20,3)>
What changes were proposed in this PR?
Ensured proper vector filling considering it's page data type.
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #4073
---
.../metadata/datatype/DecimalConverterFactory.java | 125 ++++++++-------------
.../PrestoTestUsingSparkStore.scala | 35 ++++++
.../dataload/SparkStoreCreatorForPresto.scala | 20 ++++
3 files changed, 99 insertions(+), 81 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
index 2eff647..4cdf818 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -132,6 +132,8 @@ public final class DecimalConverterFactory {
}
int shortSizeInBytes = DataTypes.SHORT.getSizeInBytes();
int intSizeInBytes = DataTypes.INT.getSizeInBytes();
+ int longSizeInBytes = DataTypes.LONG.getSizeInBytes();
+ int shortIntSizeInBytes = DataTypes.SHORT_INT.getSizeInBytes();
int lengthStoredInBytes;
if (vectorInfo.encodings != null && vectorInfo.encodings.size() > 0 &&
CarbonUtil
.hasEncoding(vectorInfo.encodings,
Encoding.INT_LENGTH_COMPLEX_CHILD_BYTE_ARRAY)) {
@@ -142,100 +144,57 @@ public final class DecimalConverterFactory {
lengthStoredInBytes = shortSizeInBytes;
}
byte[] data = (byte[]) valuesToBeConverted;
- if (pageType == DataTypes.BYTE) {
+ if (pageType == DataTypes.BYTE || pageType == DataTypes.SHORT
+ || pageType == DataTypes.SHORT_INT || pageType == DataTypes.INT
+ || pageType == DataTypes.LONG) {
for (int i = 0; i < size; i++) {
if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
- BigDecimal value = BigDecimal.valueOf(data[i], scale);
- if (value.scale() < newMeasureScale) {
- value = value.setScale(newMeasureScale);
+ BigDecimal value;
+ if (pageType == DataTypes.BYTE) {
+ value = BigDecimal.valueOf(data[i], scale);
+ } else if (pageType == DataTypes.SHORT) {
+ value = BigDecimal
+ .valueOf(ByteUtil.toShortLittleEndian(data, i *
shortSizeInBytes), scale);
+ } else if (pageType == DataTypes.SHORT_INT) {
+ value =
+ BigDecimal.valueOf(ByteUtil.valueOf3Bytes(data, i *
shortIntSizeInBytes), scale);
+ } else if (pageType == DataTypes.INT) {
+ value =
+ BigDecimal.valueOf(ByteUtil.toIntLittleEndian(data, i *
intSizeInBytes), scale);
+ } else {
+ value =
+ BigDecimal.valueOf(ByteUtil.toLongLittleEndian(data, i *
longSizeInBytes), scale);
}
- vector.putDecimal(i, value, precision);
- }
- }
- } else if (pageType == DataTypes.SHORT) {
- for (int i = 0; i < size; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(i);
- } else {
- BigDecimal value = BigDecimal
- .valueOf(ByteUtil.toShortLittleEndian(data, i *
shortSizeInBytes),
- scale);
if (value.scale() < newMeasureScale) {
value = value.setScale(newMeasureScale);
}
vector.putDecimal(i, value, precision);
}
}
- } else if (pageType == DataTypes.SHORT_INT) {
- int shortIntSizeInBytes = DataTypes.SHORT_INT.getSizeInBytes();
- for (int i = 0; i < size; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(i);
+ } else if (pageType == DataTypes.BYTE_ARRAY) {
+ // complex primitive decimal dimension
+ int offset = 0;
+ int length;
+ for (int j = 0; j < size; j++) {
+ // here decimal data will be Length[4 byte], scale[1 byte],
value[Length byte]
+ if (lengthStoredInBytes == intSizeInBytes) {
+ length = ByteBuffer.wrap(data, offset,
lengthStoredInBytes).getInt();
} else {
- BigDecimal value = BigDecimal
- .valueOf(ByteUtil.valueOf3Bytes(data, i * shortIntSizeInBytes),
- scale);
- if (value.scale() < newMeasureScale) {
- value = value.setScale(newMeasureScale);
- }
- vector.putDecimal(i, value, precision);
+ length = ByteBuffer.wrap(data, offset,
lengthStoredInBytes).getShort();
}
- }
- } else {
- if (pageType == DataTypes.INT) {
- for (int i = 0; i < size; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(i);
- } else {
- BigDecimal value = BigDecimal
- .valueOf(ByteUtil.toIntLittleEndian(data, i *
intSizeInBytes),
- scale);
- if (value.scale() < newMeasureScale) {
- value = value.setScale(newMeasureScale);
- }
- vector.putDecimal(i, value, precision);
- }
+ offset += lengthStoredInBytes;
+ if (length == 0) {
+ vector.putNull(j);
+ continue;
}
- } else if (pageType == DataTypes.LONG) {
- int longSizeInBytes = DataTypes.LONG.getSizeInBytes();
- for (int i = 0; i < size; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(i);
- } else {
- BigDecimal value = BigDecimal
- .valueOf(ByteUtil.toLongLittleEndian(data, i *
longSizeInBytes),
- scale);
- if (value.scale() < newMeasureScale) {
- value = value.setScale(newMeasureScale);
- }
- vector.putDecimal(i, value, precision);
- }
- }
- } else if (pageType == DataTypes.BYTE_ARRAY) {
- // complex primitive decimal dimension
- int offset = 0;
- int length;
- for (int j = 0; j < size; j++) {
- // here decimal data will be Length[4 byte], scale[1 byte],
value[Length byte]
- if (lengthStoredInBytes == intSizeInBytes) {
- length = ByteBuffer.wrap(data, offset,
lengthStoredInBytes).getInt();
- } else {
- length = ByteBuffer.wrap(data, offset,
lengthStoredInBytes).getShort();
- }
- offset += lengthStoredInBytes;
- if (length == 0) {
- vector.putNull(j);
- continue;
- }
- BigDecimal value = DataTypeUtil.byteToBigDecimal(data, offset,
length);
- if (value.scale() < newMeasureScale) {
- value = value.setScale(newMeasureScale);
- }
- vector.putDecimal(j, value, precision);
- offset += length;
+ BigDecimal value = DataTypeUtil.byteToBigDecimal(data, offset,
length);
+ if (value.scale() < newMeasureScale) {
+ value = value.setScale(newMeasureScale);
}
+ vector.putDecimal(j, value, precision);
+ offset += length;
}
}
}
@@ -278,7 +237,7 @@ public final class DecimalConverterFactory {
}
}
- public class DecimalUnscaledConverter implements DecimalConverter {
+ public class DecimalUnscaledConverter extends DecimalIntConverter implements
DecimalConverter {
private int scale;
@@ -287,6 +246,7 @@ public final class DecimalConverterFactory {
private byte[] decimalBuffer = new byte[minBytesForPrecision[38]];
DecimalUnscaledConverter(int precision, int scale) {
+ super(scale);
this.scale = scale;
this.numBytes = minBytesForPrecision[precision];
}
@@ -327,8 +287,11 @@ public final class DecimalConverterFactory {
@Override
public void fillVector(Object valuesToBeConverted, int size,
ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
+ if (valuesToBeConverted instanceof byte[]) {
+ super.fillVector(valuesToBeConverted, size, vectorInfo, nullBitSet,
pageType);
+ return;
+ }
CarbonColumnVector vector = getCarbonColumnVector(vectorInfo,
nullBitSet);
- //TODO handle complex child
int precision = vectorInfo.measure.getMeasure().getPrecision();
int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
if (scale < newMeasureScale) {
diff --git
a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestUsingSparkStore.scala
b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestUsingSparkStore.scala
index 345a855..2e44dd7 100644
---
a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestUsingSparkStore.scala
+++
b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestUsingSparkStore.scala
@@ -20,6 +20,7 @@ package org.apache.carbondata.presto.integrationtest
import java.io.{File}
import java.util
+import io.prestosql.jdbc.PrestoArray
import org.apache.commons.io.FileUtils
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, FunSuiteLike}
@@ -287,6 +288,7 @@ class PrestoTestUsingSparkStore
}
test("Test range columns") {
+ prestoServer.execute("drop table if exists presto_spark_db.range_table")
prestoServer
.execute(
"create table presto_spark_db.range_table(name varchar, id int) with" +
@@ -304,6 +306,7 @@ class PrestoTestUsingSparkStore
}
test("Test streaming ") {
+ prestoServer.execute("drop table if exists
presto_spark_db.streaming_table")
prestoServer
.execute(
"create table presto_spark_db.streaming_table(c1 varchar, c2 int, c3
varchar, c5 varchar)" +
@@ -325,4 +328,36 @@ class PrestoTestUsingSparkStore
}
+ test("Test decimal unscaled converter for array") {
+ prestoServer.execute("drop table if exists presto_spark_db.array_decimal")
+ prestoServer
+ .execute(
+ "create table presto_spark_db.array_decimal(salary
array(decimal(20,3)) ) with" +
+ "(format='CARBON') ")
+ copyStoreContents("array_decimal")
+ val result: List[Map[String, Any]] = prestoServer
+ .executeQuery("SELECT * FROM presto_spark_db.array_decimal")
+ assert(result.size == 1)
+ val data = result(0)("salary")
+ .asInstanceOf[PrestoArray]
+ .getArray()
+ .asInstanceOf[Array[Object]]
+ assert(data.sameElements(Array("922.580", "3.435")))
+ }
+
+ test("Test decimal unscaled converter for struct") {
+ prestoServer.execute("drop table if exists presto_spark_db.struct_decimal")
+ prestoServer
+ .execute(
+ "create table presto_spark_db.struct_decimal(salary ROW(dec
decimal(20,3))) " +
+ "with (format='CARBON') ")
+ copyStoreContents("struct_decimal")
+ val result: List[Map[String, Any]] = prestoServer
+ .executeQuery("SELECT * FROM presto_spark_db.struct_decimal")
+ assert(result.size == 1)
+ val data = result(0)("salary").asInstanceOf[java.util.Map[String, Any]]
+ assert(data.get("dec") == "922.580")
+
+ }
+
}
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkStoreCreatorForPresto.scala
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkStoreCreatorForPresto.scala
index 112218f..3f3ae3c 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkStoreCreatorForPresto.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkStoreCreatorForPresto.scala
@@ -90,6 +90,8 @@ class SparkStoreCreatorForPresto extends QueryTest with
BeforeAndAfterAll{
sql("drop table if exists carbon_bloom")
sql("drop table if exists range_table")
sql("drop table if exists streaming_table")
+ sql("drop table if exists array_decimal")
+ sql("drop table if exists struct_decimal")
sql("use default ")
}
@@ -365,6 +367,24 @@ class SparkStoreCreatorForPresto extends QueryTest with
BeforeAndAfterAll{
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO TABLE
streaming_table""")
}
+ test("Test decimal unscaled converter for array") {
+ sql("drop table if exists array_decimal")
+ sql(
+ "CREATE TABLE IF NOT EXISTS array_decimal (salary array<decimal(20,3)>)
STORED AS " +
+ "carbondata"
+ )
+ sql("insert into array_decimal select array(922.580, 3.435) ")
+ }
+
+ test("Test decimal unscaled converter for struct") {
+ sql("drop table if exists struct_decimal")
+ sql(
+ "CREATE TABLE IF NOT EXISTS struct_decimal (salary
struct<dec:decimal(20,3)> ) STORED AS " +
+ "carbondata"
+ )
+ sql("insert into struct_decimal select named_struct('dec',922.580) ")
+ }
+
private def createFile(fileName: String, line: Int = 10000, start: Int = 0)
= {
if (!new File(fileName).exists()) {
val write = new PrintWriter(new File(fileName))
