[02/28] carbondata git commit: [CARBONDATA-1720][FILTER] Wrong data displayed for <= filter for timestamp column(dictionary column)

2017-11-18 Thread jackylk 
[CARBONDATA-1720][FILTER] Wrong data displayed for <= filter for timestamp 
column(dictionary column)

Issue:
<= filter is giving wrong results for timestamp dictioinary column
Solution:
In less than equal to filter, we are considering surrogate 2 as default value. 
But surrogate 1 is for default value.

This closes #1502


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/17892b17
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/17892b17
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/17892b17

Branch: refs/heads/fgdatamap
Commit: 17892b17b688eaa637b3dd97c25286edb4183eaa
Parents: 09d0205
Author: dhatchayani 
Authored: Wed Nov 15 18:41:00 2017 +0530
Committer: ravipesala 
Committed: Thu Nov 16 16:12:19 2017 +0530

--
 .../executer/RangeValueFilterExecuterImpl.java  |   2 +-
 ...velRangeLessThanEqualFilterExecuterImpl.java |  10 +-
 .../RowLevelRangeLessThanFiterExecuterImpl.java |  14 +-
 .../src/test/resources/timestamp.csv| 301 +++
 .../RangeFilterAllDataTypesTestCases.scala  |   9 +
 5 files changed, 327 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 0cfa198..ee373c5 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -554,7 +554,7 @@ public class RangeValueFilterExecuterImpl extends 
ValueBasedFilterExecuterImpl {
   if 
(dimColEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
 DirectDictionaryGenerator directDictionaryGenerator = 
DirectDictionaryKeyGeneratorFactory
 
.getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType());
-int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 
1;
+int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
 CarbonDimension currentBlockDimension =
 segmentProperties.getDimensions().get(dimensionBlocksIndex);
 if (currentBlockDimension.isSortColumn()) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 5e0bb41..88cf75c 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -267,7 +267,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl 
extends RowLevelFilter
   DirectDictionaryGenerator directDictionaryGenerator = 
DirectDictionaryKeyGeneratorFactory
   .getDirectDictionaryGenerator(
   dimColEvaluatorInfoList.get(0).getDimension().getDataType());
-  int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+  int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
   CarbonDimension currentBlockDimension =
   segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
   if (currentBlockDimension.isSortColumn()) {
@@ -324,7 +324,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl 
extends RowLevelFilter
   return bitSet;
 }
   } else {
-skip = start;
+// as start will be last index of null value inclusive
+// so adding 1 to skip last null value
+skip = start + 1;
   }
   startIndex = skip;
 }
@@ -392,7 +394,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl 
extends RowLevelFilter
 return bitSet;
   }
 } else {
-  skip = start;
+  // as start will be last index of null value inclusive
+  // so adding 1 to skip last null value
+  skip = start + 1;
 }
 startIndex = skip;
   }

carbondata git commit: [CARBONDATA-1720][FILTER] Wrong data displayed for <= filter for timestamp column(dictionary column)

2017-11-16 Thread ravipesala 
Repository: carbondata
Updated Branches:
  refs/heads/master 09d020561 -> 17892b17b


[CARBONDATA-1720][FILTER] Wrong data displayed for <= filter for timestamp 
column(dictionary column)

Issue:
<= filter is giving wrong results for timestamp dictioinary column
Solution:
In less than equal to filter, we are considering surrogate 2 as default value. 
But surrogate 1 is for default value.

This closes #1502


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/17892b17
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/17892b17
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/17892b17

Branch: refs/heads/master
Commit: 17892b17b688eaa637b3dd97c25286edb4183eaa
Parents: 09d0205
Author: dhatchayani 
Authored: Wed Nov 15 18:41:00 2017 +0530
Committer: ravipesala 
Committed: Thu Nov 16 16:12:19 2017 +0530

--
 .../executer/RangeValueFilterExecuterImpl.java  |   2 +-
 ...velRangeLessThanEqualFilterExecuterImpl.java |  10 +-
 .../RowLevelRangeLessThanFiterExecuterImpl.java |  14 +-
 .../src/test/resources/timestamp.csv| 301 +++
 .../RangeFilterAllDataTypesTestCases.scala  |   9 +
 5 files changed, 327 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 0cfa198..ee373c5 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -554,7 +554,7 @@ public class RangeValueFilterExecuterImpl extends 
ValueBasedFilterExecuterImpl {
   if 
(dimColEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
 DirectDictionaryGenerator directDictionaryGenerator = 
DirectDictionaryKeyGeneratorFactory
 
.getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType());
-int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 
1;
+int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
 CarbonDimension currentBlockDimension =
 segmentProperties.getDimensions().get(dimensionBlocksIndex);
 if (currentBlockDimension.isSortColumn()) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 5e0bb41..88cf75c 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -267,7 +267,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl 
extends RowLevelFilter
   DirectDictionaryGenerator directDictionaryGenerator = 
DirectDictionaryKeyGeneratorFactory
   .getDirectDictionaryGenerator(
   dimColEvaluatorInfoList.get(0).getDimension().getDataType());
-  int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+  int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
   CarbonDimension currentBlockDimension =
   segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
   if (currentBlockDimension.isSortColumn()) {
@@ -324,7 +324,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl 
extends RowLevelFilter
   return bitSet;
 }
   } else {
-skip = start;
+// as start will be last index of null value inclusive
+// so adding 1 to skip last null value
+skip = start + 1;
   }
   startIndex = skip;
 }
@@ -392,7 +394,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl 
extends RowLevelFilter
 return bitSet;
   }
 } else {
-  skip = start;
+  // as start will be last index of null value inclusive
+  // so adding 1 to skip last null value
+