[CARBONDATA-1720][FILTER] Wrong data displayed for <= filter for timestamp column(dictionary column)
Issue: <= filter is giving wrong results for timestamp dictioinary column Solution: In less than equal to filter, we are considering surrogate 2 as default value. But surrogate 1 is for default value. This closes #1502 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/17892b17 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/17892b17 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/17892b17 Branch: refs/heads/fgdatamap Commit: 17892b17b688eaa637b3dd97c25286edb4183eaa Parents: 09d0205 Author: dhatchayani <dhatcha.offic...@gmail.com> Authored: Wed Nov 15 18:41:00 2017 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Thu Nov 16 16:12:19 2017 +0530 ---------------------------------------------------------------------- .../executer/RangeValueFilterExecuterImpl.java | 2 +- ...velRangeLessThanEqualFilterExecuterImpl.java | 10 +- .../RowLevelRangeLessThanFiterExecuterImpl.java | 14 +- .../src/test/resources/timestamp.csv | 301 +++++++++++++++++++ .../RangeFilterAllDataTypesTestCases.scala | 9 + 5 files changed, 327 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java index 0cfa198..ee373c5 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java @@ -554,7 +554,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { if (dimColEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType()); - int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1; + int key = directDictionaryGenerator.generateDirectSurrogateKey(null); CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dimensionBlocksIndex); if (currentBlockDimension.isSortColumn()) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java index 5e0bb41..88cf75c 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java @@ -267,7 +267,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator( dimColEvaluatorInfoList.get(0).getDimension().getDataType()); - int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1; + int key = directDictionaryGenerator.generateDirectSurrogateKey(null); CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dimensionBlocksIndex[0]); if (currentBlockDimension.isSortColumn()) { @@ -324,7 +324,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter return bitSet; } } else { - skip = start; + // as start will be last index of null value inclusive + // so adding 1 to skip last null value + skip = start + 1; } startIndex = skip; } @@ -392,7 +394,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter return bitSet; } } else { - skip = start; + // as start will be last index of null value inclusive + // so adding 1 to skip last null value + skip = start + 1; } startIndex = skip; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java index 74162f6..7f735c2 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java @@ -269,7 +269,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator( dimColEvaluatorInfoList.get(0).getDimension().getDataType()); - int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1; + int key = directDictionaryGenerator.generateDirectSurrogateKey(null); CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dimensionBlocksIndex[0]); if (currentBlockDimension.isSortColumn()) { @@ -319,7 +319,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut if (null != defaultValue) { start = CarbonUtil .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, - defaultValue, false); + defaultValue, true); if (start < 0) { skip = -(start + 1); // end of block @@ -327,7 +327,9 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut return bitSet; } } else { - skip = start; + // as start will be last index of null value inclusive + // so adding 1 to skip last null value + skip = start + 1; } startIndex = skip; } @@ -395,7 +397,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut if (null != defaultValue) { start = CarbonUtil .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, - numerOfRows - 1, defaultValue, false); + numerOfRows - 1, defaultValue, true); if (start < 0) { skip = -(start + 1); // end of block @@ -403,7 +405,9 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut return bitSet; } } else { - skip = start; + // as start will be last index of null value inclusive + // so adding 1 to skip last null value + skip = start + 1; } startIndex = skip; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/integration/spark-common-test/src/test/resources/timestamp.csv ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/resources/timestamp.csv b/integration/spark-common-test/src/test/resources/timestamp.csv new file mode 100644 index 0000000..0a7b3ff --- /dev/null +++ b/integration/spark-common-test/src/test/resources/timestamp.csv @@ -0,0 +1,301 @@ +timestampCol +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 +1970-01-01 05:30:00 http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala index 00be125..87cc194 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala @@ -263,6 +263,15 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll { ) } + test("test range filter for less than filter"){ + sql("drop table if exists timestampTable") + sql("create table timestampTable (timestampCol timestamp) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='timestampCol')") + sql(s"load data local inpath '$resourcesPath/timestamp.csv' into table timestampTable") + checkAnswer(sql("select * from timestampTable where timestampCol='1970-01-01 05:30:00'"), + sql("select * from timestampTable where timestampCol<='1970-01-01 05:30:00'")) + sql("drop table if exists timestampTable") + } + test("test range filter for direct dictionary not equality"){ checkAnswer( sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'"),