[CARBONDATA-2031] Fix ArrayIndexOutOfBoundException when filter query is applied on column where all values are null and column is noinverted index column
Fix ArrayIndexOutOfBoundException when filter query is applied on column where all values are null and column is noinverted index column.when all the values are null in no inverted index column the number of exclude filter keys are null, hence just return the bitset if the exclude filters to be applied are none. This closes #1809 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/383b2ed3 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/383b2ed3 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/383b2ed3 Branch: refs/heads/carbonstore Commit: 383b2ed357f93c0a18fcbfb85e36e7156a0ee610 Parents: 4f9aeae Author: akashrn5 <[email protected]> Authored: Tue Jan 16 12:17:53 2018 +0530 Committer: kumarvishal <[email protected]> Committed: Wed Jan 17 19:08:41 2018 +0530 ---------------------------------------------------------------------- .../scan/filter/executer/ExcludeFilterExecuterImpl.java | 4 ++++ .../dataload/TestNoInvertedIndexLoadAndQuery.scala | 9 +++++++++ 2 files changed, 13 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/383b2ed3/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java index f680579..465bee6 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java @@ -378,6 +378,10 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { BitSet bitSet = new BitSet(numerOfRows); bitSet.flip(0, numerOfRows); byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys(); + // filterValues can be null when the dictionary chunk and surrogate size both are one + if (filterValues.length == 0) { + return bitSet; + } // binary search can only be applied if column is sorted if (isNaturalSorted) { int startIndex = 0; http://git-wip-us.apache.org/repos/asf/carbondata/blob/383b2ed3/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala index baf3997..e8affc4 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala @@ -49,6 +49,7 @@ class TestNoInvertedIndexLoadAndQuery extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS index2") sql("DROP TABLE IF EXISTS hiveNoInvertedIndexTable") sql("DROP TABLE IF EXISTS carbonNoInvertedIndexTable") + sql("DROP TABLE IF EXISTS testNull") } test("no inverted index load and point query") { @@ -281,10 +282,18 @@ class TestNoInvertedIndexLoadAndQuery extends QueryTest with BeforeAndAfterAll { true,"NOINVERTEDINDEX") } + test("filter query on dictionary and no inverted index column where all values are null"){ + sql("""create table testNull (c1 string,c2 int,c3 string,c5 string) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='C2','NO_INVERTED_INDEX'='C2')""") + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table testNull OPTIONS('delimiter'=';','fileheader'='c1,c2,c3,c5')""") + sql("""select c2 from testNull where c2 is null""").show() + checkAnswer(sql("""select c2 from testNull where c2 is null"""), Seq(Row(null), Row(null), Row(null), Row(null), Row(null), Row(null))) + } + override def afterAll { sql("drop table if exists index1") sql("drop table if exists index2") sql("drop table if exists indexFormat") + sql("drop table if exists testNull") clean }
