This is an automated email from the ASF dual-hosted git repository. ajantha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 9f461b7 [CARBONDATA-3965] Fixed float variable target datatype in case of adaptive encoding 9f461b7 is described below commit 9f461b747628495517144810240bcb8656d18498 Author: Nihal ojha <nihalnit...@gmail.com> AuthorDate: Thu Oct 15 16:29:33 2020 +0530 [CARBONDATA-3965] Fixed float variable target datatype in case of adaptive encoding Why is this PR needed? Currently, float variables are using long value 8 bytes to store float data. What changes were proposed in this PR? Handled the float variables to consider the target data type based on the given value. Does this PR introduce any user interface change? No Is any new testcase added? Yes This closes #3985 --- .../page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java | 7 ------- .../datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java | 6 ------ .../datastore/page/statistics/PrimitivePageStatsCollector.java | 2 +- .../spark/testsuite/complexType/TestAdaptiveComplexType.scala | 9 +++++++++ 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java index 4928646..f9abd16 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java @@ -288,13 +288,6 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec { for (int i = 0; i < size; i += intSizeInBytes) { vector.putFloat(rowId++, (max - ByteUtil.toIntLittleEndian(pageData, i)) / floatFactor); } - } else if (pageDataType == DataTypes.LONG) { - // complex primitive float type can enter here. - int size = pageSize * longSizeInBytes; - for (int i = 0; i < size; i += longSizeInBytes) { - vector.putFloat(rowId++, - (float) ((max - ByteUtil.toLongLittleEndian(pageData, i)) / factor)); - } } else { throw new RuntimeException("internal error: " + this.toString()); } diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java index d07eaa7..c6a0c28 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java @@ -276,12 +276,6 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec { for (int i = 0; i < size; i += intSizeInBytes) { vector.putFloat(rowId++, (ByteUtil.toIntLittleEndian(pageData, i) / floatFactor)); } - } else if (pageDataType == DataTypes.LONG) { - // complex primitive float type can enter here. - int size = pageSize * longSizeInBytes; - for (int i = 0; i < size; i += longSizeInBytes) { - vector.putFloat(rowId++, (float) (ByteUtil.toLongLittleEndian(pageData, i) / factor)); - } } else { throw new RuntimeException("internal error: " + this.toString()); } diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java index 190a8d1..b301cac 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java @@ -256,7 +256,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si } private int getDecimalCount(float value) { - return getDecimalCount((double) value); + return getDecimalCount(Double.parseDouble(Float.toString(value))); } @Override diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala index ca24a07..8a4fcd5 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala @@ -574,4 +574,13 @@ trait TestAdaptiveComplexType extends QueryTest with BeforeAndAfterAll { "5555555.9559, 1.2345678991234568E16, 3444.999") } + test("test adaptive encoding with float as complex primitive, Encoding FLOAT --> SHORT") { + sql("drop table if exists floatComplexPrimitive") + sql("create table floatComplexPrimitive (arrayField array<float>) using carbon ") + sql("insert into floatComplexPrimitive values (array(null, 5.121))") + checkAnswer(sql("select * from floatComplexPrimitive"), + Seq(Row(mutable.WrappedArray.make(Array(null, 5.121f))))) + sql("drop table if exists floatComplexPrimitive") + } + }