Repository: carbondata Updated Branches: refs/heads/master 02a89a9c6 -> 61fcdf286
[CARBONDATA-2940]Fix BufferUnderFlowException for ComplexPushDown Problem: BufferUnderFlow Exception was thrown, as buffer data was filled with wrong values. Solution: Check for if column is of Primitive Dimension column with ComplexParentOrdinal = -1 and not Complex and increment Buffer index for NoDictionaryKeys. This closes #2727 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/61fcdf28 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/61fcdf28 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/61fcdf28 Branch: refs/heads/master Commit: 61fcdf286cabfce199b402b63ffc55bbf5cc64de Parents: 02a89a9 Author: Indhumathi27 <[email protected]> Authored: Mon Sep 17 12:32:49 2018 +0530 Committer: kunal642 <[email protected]> Committed: Tue Sep 18 17:29:49 2018 +0530 ---------------------------------------------------------------------- .../core/datastore/blocklet/BlockletEncodedColumnPage.java | 2 +- .../core/scan/collector/impl/DictionaryBasedResultCollector.java | 3 +++ .../spark/testsuite/complexType/TestComplexDataType.scala | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java index 135b1e2..da4e211 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java @@ -213,7 +213,7 @@ public class BlockletEncodedColumnPage { new DecoderBasedFallbackEncoder(encodedColumnPage, pageIndex, localDictionaryGenerator))); } else { fallbackFutureQueue.add(fallbackExecutorService.submit( - new ActualDataBasedFallbackEncoder(encodedColumnPage, encodedColumnPageList.size()))); + new ActualDataBasedFallbackEncoder(encodedColumnPage, pageIndex))); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java index 1faf2fd..69b142f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java @@ -231,6 +231,9 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect childColumnByteBuffer .put(queryDimensions[i].getDimension(), buffer); mergedComplexDimensionDataMap.put(complexParentOrdinal, childColumnByteBuffer); + } else if (!queryDimensions[i].getDimension().isComplex()) { + // If Dimension is not a Complex Column, then increment index for noDictionaryComplexColumn + noDictionaryComplexColumnIndex++; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala index 8527380..220451b 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala @@ -1015,6 +1015,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { "create table table1 (id int, name string, structField struct<intval:int, stringval:string>) stored by 'carbondata'") sql("insert into table1 values(null,'aaa','23$bb')") checkAnswer(sql("select * from table1"),Seq(Row(null,"aaa", Row(23,"bb")))) + checkAnswer(sql("select id,name,structField.intval,structField.stringval from table1"),Seq(Row(null,"aaa",23,"bb"))) + checkAnswer(sql("select id,name,structField.intval,structField.stringval,name from table1"),Seq(Row(null,"aaa",23,"bb","aaa"))) + checkAnswer(sql("select id,name,structField.intval,name,structField.stringval from table1"),Seq(Row(null,"aaa",23,"aaa","bb"))) } }
