Repository: incubator-carbondata Updated Branches: refs/heads/master f88ab66a9 -> 0e82b6501
[CARBONDATA-278] IS NULL and IS NOT NULL shall be push down to carbon layer since carbon layer can process these filters faster using block/block-let pruning , also while processing filters in executors carbon is applying binary search for applying filter values. Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e5f1a99b Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e5f1a99b Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e5f1a99b Branch: refs/heads/master Commit: e5f1a99ba3c02655d769352474a4b85c66821a0a Parents: f88ab66 Author: sujith71955 <sujithchacko.2...@gmail.com> Authored: Tue Sep 27 16:51:03 2016 +0530 Committer: jackylk <jacky.li...@huawei.com> Committed: Mon Oct 24 20:42:24 2016 +0800 ---------------------------------------------------------------------- .../conditional/BinaryConditionalExpression.java | 2 +- .../scan/expression/conditional/EqualToExpression.java | 1 - .../carbondata/scan/filter/FilterExpressionProcessor.java | 3 ++- .../filter/resolver/ConditionalFilterResolverImpl.java | 3 ++- .../scala/org/apache/carbondata/spark/CarbonFilters.scala | 10 ++++++++-- 5 files changed, 13 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5f1a99b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java index 3d7e15e..9db6953 100644 --- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java +++ b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java @@ -29,7 +29,7 @@ public abstract class BinaryConditionalExpression extends BinaryLogicalExpressio * */ private static final long serialVersionUID = 1L; - + public boolean isNull; public BinaryConditionalExpression(Expression left, Expression right) { super(left, right); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5f1a99b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java index 8f7fa0a..ddb0456 100644 --- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java +++ b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java @@ -31,7 +31,6 @@ import org.apache.carbondata.scan.filter.intf.RowIntf; public class EqualToExpression extends BinaryConditionalExpression { private static final long serialVersionUID = 1L; - private boolean isNull; public EqualToExpression(Expression left, Expression right) { super(left, right); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5f1a99b/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java index b50e6e6..1541867 100644 --- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java +++ b/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java @@ -232,7 +232,8 @@ public class FilterExpressionProcessor implements FilterProcessor { currentExpression), currentExpression); case EQUALS: case IN: - return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, false, expressionTree, + return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, + ((BinaryConditionalExpression) expressionTree).isNull, expressionTree, tableIdentifier, expressionTree); case GREATERTHAN: case GREATERTHAN_EQUALTO: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5f1a99b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java index bcb0a1b..612ea6f 100644 --- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java @@ -132,7 +132,8 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { metadata.setColumnExpression(columnList.get(0)); metadata.setExpression(exp); metadata.setIncludeFilter(isIncludeFilter); - if (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) { + if (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) + || columnList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType( FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0)), metadata); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5f1a99b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala index 5fb0051..03c6053 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala @@ -153,7 +153,8 @@ object CarbonFilters { Some(sources.Not(sources.EqualTo(a.name, v))) case Not(EqualTo(Literal(v, t), Cast(a: Attribute, _))) => new Some(sources.Not(sources.EqualTo(a.name, v))) - + case IsNotNull(a: Attribute) => Some(sources.IsNotNull(a.name)) + case IsNull(a: Attribute) => Some(sources.IsNull(a.name)) case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) => val hSet = list.map(e => e.eval(EmptyRow)) Some(sources.Not(sources.In(a.name, hSet.toArray))) @@ -257,7 +258,12 @@ object CarbonFilters { Some(new NotEqualsExpression(transformExpression(a).get, transformExpression(l).get)) case Not(EqualTo(l@Literal(v, t), Cast(a: Attribute, _))) => new Some(new NotEqualsExpression(transformExpression(a).get, transformExpression(l).get)) - + case IsNotNull(child) => + Some(new NotEqualsExpression(transformExpression(child).get, + transformExpression(Literal(null)).get, true)) + case IsNull(child) => + Some(new EqualToExpression(transformExpression(child).get, + transformExpression(Literal(null)).get, true)) case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) => if (list.exists(x => (isNullLiteral(x.asInstanceOf[Literal])))) {
