[carbondata] branch master updated: [CARBONDATA-3965] Fixed float variable target datatype in case of adaptive encoding

[ajantha]

This is an automated email from the ASF dual-hosted git repository.

ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 9f461b7  [CARBONDATA-3965] Fixed float variable target datatype in 
case of adaptive encoding
9f461b7 is described below

commit 9f461b747628495517144810240bcb8656d18498
Author: Nihal ojha <nihalnit...@gmail.com>
AuthorDate: Thu Oct 15 16:29:33 2020 +0530

    [CARBONDATA-3965] Fixed float variable target datatype in case of adaptive 
encoding
    
    Why is this PR needed?
    Currently, float variables are using long value 8 bytes to store float data.
    
    What changes were proposed in this PR?
    Handled the float variables to consider the target data type based on the 
given value.
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    Yes
    
    This closes #3985
---
 .../page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java       | 7 -------
 .../datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java  | 6 ------
 .../datastore/page/statistics/PrimitivePageStatsCollector.java   | 2 +-
 .../spark/testsuite/complexType/TestAdaptiveComplexType.scala    | 9 +++++++++
 4 files changed, 10 insertions(+), 14 deletions(-)

diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
index 4928646..f9abd16 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
@@ -288,13 +288,6 @@ public class AdaptiveDeltaFloatingCodec extends 
AdaptiveCodec {
           for (int i = 0; i < size; i += intSizeInBytes) {
             vector.putFloat(rowId++, (max - 
ByteUtil.toIntLittleEndian(pageData, i)) / floatFactor);
           }
-        } else if (pageDataType == DataTypes.LONG) {
-          // complex primitive float type can enter here.
-          int size = pageSize * longSizeInBytes;
-          for (int i = 0; i < size; i += longSizeInBytes) {
-            vector.putFloat(rowId++,
-                (float) ((max - ByteUtil.toLongLittleEndian(pageData, i)) / 
factor));
-          }
         } else {
           throw new RuntimeException("internal error: " + this.toString());
         }
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index d07eaa7..c6a0c28 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -276,12 +276,6 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
           for (int i = 0; i < size; i += intSizeInBytes) {
             vector.putFloat(rowId++, (ByteUtil.toIntLittleEndian(pageData, i) 
/ floatFactor));
           }
-        } else if (pageDataType == DataTypes.LONG) {
-          // complex primitive float type can enter here.
-          int size = pageSize * longSizeInBytes;
-          for (int i = 0; i < size; i += longSizeInBytes) {
-            vector.putFloat(rowId++, (float) 
(ByteUtil.toLongLittleEndian(pageData, i) / factor));
-          }
         } else {
           throw new RuntimeException("internal error: " + this.toString());
         }
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 190a8d1..b301cac 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -256,7 +256,7 @@ public class PrimitivePageStatsCollector implements 
ColumnPageStatsCollector, Si
   }
 
   private int getDecimalCount(float value) {
-    return getDecimalCount((double) value);
+    return getDecimalCount(Double.parseDouble(Float.toString(value)));
   }
 
   @Override
diff --git 
a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
 
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
index ca24a07..8a4fcd5 100644
--- 
a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
+++ 
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
@@ -574,4 +574,13 @@ trait TestAdaptiveComplexType extends QueryTest with 
BeforeAndAfterAll {
       "5555555.9559, 1.2345678991234568E16, 3444.999")
   }
 
+  test("test adaptive encoding with float as complex primitive, Encoding FLOAT 
--> SHORT") {
+    sql("drop table if exists floatComplexPrimitive")
+    sql("create table floatComplexPrimitive (arrayField array<float>) using 
carbon ")
+    sql("insert into floatComplexPrimitive values (array(null, 5.121))")
+    checkAnswer(sql("select * from floatComplexPrimitive"),
+      Seq(Row(mutable.WrappedArray.make(Array(null, 5.121f)))))
+    sql("drop table if exists floatComplexPrimitive")
+  }
+
 }