Repository: carbondata Updated Branches: refs/heads/master 5c058acc7 -> 9ee74fe07
[CARBONDATA-2343][DataMap]Improper filter resolver cause more filter scan on data that could be skipped Currently DataMapChooser will choose and combine datamap for expressions and it will wrap the expression in a TrueConditionalResolverImpl. However the executor TrueFilterExecutor will always cause scanning the blocklet which could be skipped. This closes #2168 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9ee74fe0 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9ee74fe0 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9ee74fe0 Branch: refs/heads/master Commit: 9ee74fe0708afead309540777c315e0d13abc010 Parents: 5c058ac Author: xuchuanyin <xuchuan...@hust.edu.cn> Authored: Fri Apr 13 15:14:25 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Fri Apr 13 23:42:23 2018 +0800 ---------------------------------------------------------------------- .../carbondata/core/datamap/DataMapChooser.java | 39 ++++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ee74fe0/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java index ac00e71..a40644a 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java @@ -34,7 +34,6 @@ import org.apache.carbondata.core.scan.expression.logical.AndExpression; import org.apache.carbondata.core.scan.expression.logical.OrExpression; import org.apache.carbondata.core.scan.filter.intf.ExpressionType; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; -import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.TrueConditionalResolverImpl; /** * This chooser does 2 jobs. @@ -75,12 +74,12 @@ public class DataMapChooser { // First check for FG datamaps if any exist List<TableDataMap> allDataMapFG = DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapLevel.FG); - ExpressionTuple tuple = selectDataMap(expression, allDataMapFG); + ExpressionTuple tuple = selectDataMap(expression, allDataMapFG, resolverIntf); if (tuple.dataMapExprWrapper == null) { // Check for CG datamap List<TableDataMap> allDataMapCG = DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapLevel.CG); - tuple = selectDataMap(expression, allDataMapCG); + tuple = selectDataMap(expression, allDataMapCG, resolverIntf); } if (tuple.dataMapExprWrapper != null) { return tuple.dataMapExprWrapper; @@ -92,13 +91,16 @@ public class DataMapChooser { resolverIntf); } - private ExpressionTuple selectDataMap(Expression expression, List<TableDataMap> allDataMap) { + private ExpressionTuple selectDataMap(Expression expression, List<TableDataMap> allDataMap, + FilterResolverIntf filterResolverIntf) { switch (expression.getFilterExpressionType()) { case AND: if (expression instanceof AndExpression) { AndExpression andExpression = (AndExpression) expression; - ExpressionTuple left = selectDataMap(andExpression.getLeft(), allDataMap); - ExpressionTuple right = selectDataMap(andExpression.getRight(), allDataMap); + ExpressionTuple left = selectDataMap(andExpression.getLeft(), allDataMap, + filterResolverIntf.getLeft()); + ExpressionTuple right = selectDataMap(andExpression.getRight(), allDataMap, + filterResolverIntf.getRight()); Set<ExpressionType> filterExpressionTypes = new HashSet<>(); // If both left and right has datamap then we can either merge both datamaps to single // datamap if possible. Otherwise apply AND expression. @@ -118,16 +120,14 @@ public class DataMapChooser { if (dataMap != null) { ExpressionTuple tuple = new ExpressionTuple(); tuple.columnExpressions = columnExpressions; - tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, - new TrueConditionalResolverImpl(expression, false, false)); + tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, filterResolverIntf); return tuple; } else { // Apply AND expression. ExpressionTuple tuple = new ExpressionTuple(); tuple.columnExpressions = columnExpressions; - tuple.dataMapExprWrapper = - new AndDataMapExprWrapper(left.dataMapExprWrapper, right.dataMapExprWrapper, - new TrueConditionalResolverImpl(expression, false, false)); + tuple.dataMapExprWrapper = new AndDataMapExprWrapper(left.dataMapExprWrapper, + right.dataMapExprWrapper, filterResolverIntf); return tuple; } } else if (left.dataMapExprWrapper != null && right.dataMapExprWrapper == null) { @@ -142,8 +142,10 @@ public class DataMapChooser { case OR: if (expression instanceof OrExpression) { OrExpression orExpression = (OrExpression) expression; - ExpressionTuple left = selectDataMap(orExpression.getLeft(), allDataMap); - ExpressionTuple right = selectDataMap(orExpression.getRight(), allDataMap); + ExpressionTuple left = selectDataMap(orExpression.getLeft(), allDataMap, + filterResolverIntf.getLeft()); + ExpressionTuple right = selectDataMap(orExpression.getRight(), allDataMap, + filterResolverIntf.getRight()); Set<ExpressionType> filterExpressionTypes = new HashSet<>(); // If both left and right has datamap then we can either merge both datamaps to single // datamap if possible. Otherwise apply OR expression. @@ -162,15 +164,13 @@ public class DataMapChooser { if (dataMap != null) { ExpressionTuple tuple = new ExpressionTuple(); tuple.columnExpressions = columnExpressions; - tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, - new TrueConditionalResolverImpl(expression, false, false)); + tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, filterResolverIntf); return tuple; } else { ExpressionTuple tuple = new ExpressionTuple(); tuple.columnExpressions = columnExpressions; - tuple.dataMapExprWrapper = - new OrDataMapExprWrapper(left.dataMapExprWrapper, right.dataMapExprWrapper, - new TrueConditionalResolverImpl(expression, false, false)); + tuple.dataMapExprWrapper = new OrDataMapExprWrapper(left.dataMapExprWrapper, + right.dataMapExprWrapper, filterResolverIntf); return tuple; } } else { @@ -187,8 +187,7 @@ public class DataMapChooser { TableDataMap dataMap = chooseDataMap(allDataMap, tuple.columnExpressions, filterExpressionTypes); if (dataMap != null) { - tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, - new TrueConditionalResolverImpl(expression, false, false)); + tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, filterResolverIntf); } return tuple; }
