Repository: carbondata Updated Branches: refs/heads/master bcf3e0fd5 -> 94a4f8314
[HOTFIX-compatibility] Handle Lazy loading with inverted index for ColumnarVectorWrapperDirectWithInvertedIndex Problem: Create a store with 1.4 code with inverted index and read it with vector filling (latest master code). below exception will be thrown from AbstractCarbonColumnarVector. UnsupportedOperationException("Not allowed from here " + getClass().getName()); cause: when the lazy loading with an inverted index, getBlockDataType() was not implemented for ColumnarVectorWrapperDirectWithInvertedIndex. So, Added implementation. This closes #2870 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/94a4f831 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/94a4f831 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/94a4f831 Branch: refs/heads/master Commit: 94a4f8314068ffd4c0743907752b58879578749b Parents: bcf3e0f Author: ajantha-bhat <ajanthab...@gmail.com> Authored: Mon Oct 29 12:33:55 2018 +0530 Committer: kumarvishal09 <kumarvishal1...@gmail.com> Committed: Wed Oct 31 17:54:03 2018 +0530 ---------------------------------------------------------------------- .../encoding/adaptive/AdaptiveDeltaFloatingCodec.java | 10 ++++++++++ .../page/encoding/adaptive/AdaptiveFloatingCodec.java | 10 ++++++++++ .../ColumnarVectorWrapperDirectWithInvertedIndex.java | 6 ++++++ 3 files changed, 26 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/94a4f831/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java index d73318d..f91ede5 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java @@ -272,6 +272,11 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec { int shortInt = ByteUtil.valueOf3Bytes(shortIntPage, i * 3); vector.putFloat(i, (max - shortInt) / floatFactor); } + } else if (pageDataType == DataTypes.INT) { + int[] intData = columnPage.getIntPage(); + for (int i = 0; i < pageSize; i++) { + vector.putFloat(i, (max - intData[i]) / floatFactor); + } } else { throw new RuntimeException("internal error: " + this.toString()); } @@ -298,6 +303,11 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec { for (int i = 0; i < pageSize; i++) { vector.putDouble(i, (max - intData[i]) / factor); } + } else if (pageDataType == DataTypes.LONG) { + long[] longData = columnPage.getLongPage(); + for (int i = 0; i < pageSize; i++) { + vector.putDouble(i, (max - longData[i]) / factor); + } } else { throw new RuntimeException("Unsupported datatype : " + pageDataType); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/94a4f831/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java index b300ee1..49696eb 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java @@ -274,6 +274,11 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec { int shortInt = ByteUtil.valueOf3Bytes(shortIntPage, i * 3); vector.putFloat(i, (shortInt / floatFactor)); } + } else if (pageDataType == DataTypes.INT) { + int[] intData = columnPage.getIntPage(); + for (int i = 0; i < pageSize; i++) { + vector.putFloat(i, (intData[i] / floatFactor)); + } } else { throw new RuntimeException("internal error: " + this.toString()); } @@ -300,6 +305,11 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec { for (int i = 0; i < pageSize; i++) { vector.putDouble(i, (intData[i] / factor)); } + } else if (pageDataType == DataTypes.LONG) { + long[] longData = columnPage.getLongPage(); + for (int i = 0; i < pageSize; i++) { + vector.putDouble(i, (longData[i] / factor)); + } } else { throw new RuntimeException("Unsupported datatype : " + pageDataType); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/94a4f831/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java index d95267f..f190d7d 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java @@ -19,6 +19,7 @@ package org.apache.carbondata.core.scan.result.vector.impl.directread; import java.math.BigDecimal; +import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; /** @@ -141,4 +142,9 @@ class ColumnarVectorWrapperDirectWithInvertedIndex extends AbstractCarbonColumna columnVector.putByte(invertedIndex[rowId++], src[i]); } } + + @Override + public DataType getBlockDataType() { + return columnVector.getBlockDataType(); + } }