Repository: drill Updated Branches: refs/heads/master a264e7feb -> 9a6cb59b9
DRILL-6259: Support parquet filter push down for complex types close apache/drill#1173 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4ee5625d Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4ee5625d Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4ee5625d Branch: refs/heads/master Commit: 4ee5625d57bd73d3d82b45f687a8574ea6660f8e Parents: a264e7f Author: Arina Ielchiieva <arina.yelchiy...@gmail.com> Authored: Tue Mar 13 19:54:25 2018 +0200 Committer: Aman Sinha <asi...@maprtech.com> Committed: Fri Mar 30 22:45:33 2018 -0700 ---------------------------------------------------------------------- .../sig/ConstantExpressionIdentifier.java | 6 + .../exec/expr/ExpressionTreeMaterializer.java | 3 +- .../exec/expr/stat/ParquetIsPredicates.java | 23 ++++ .../exec/expr/stat/RangeExprEvaluator.java | 27 ++-- .../drill/exec/expr/stat/TypedFieldExpr.java | 63 ---------- .../exec/planner/common/DrillRelOptUtil.java | 14 +-- .../logical/DrillPushFilterPastProjectRule.java | 14 ++- .../store/parquet/ParquetFilterBuilder.java | 80 ++++-------- .../store/parquet/ParquetPushDownFilter.java | 13 +- .../parquet/stat/ParquetMetaStatCollector.java | 29 ++--- ...estParquetFilterPushDownForComplexTypes.java | 124 +++++++++++++++++++ .../resources/parquet/users/users_1.parquet | Bin 0 -> 657 bytes .../resources/parquet/users/users_2.parquet | Bin 0 -> 641 bytes .../resources/parquet/users/users_3.parquet | Bin 0 -> 588 bytes .../resources/parquet/users/users_4.parquet | Bin 0 -> 653 bytes .../resources/parquet/users/users_5.parquet | Bin 0 -> 583 bytes .../resources/parquet/users/users_6.parquet | Bin 0 -> 627 bytes .../resources/parquet/users/users_7.parquet | Bin 0 -> 662 bytes .../drill/common/expression/SchemaPath.java | 29 +++-- .../drill/common/expression/TypedFieldExpr.java | 60 +++++++++ .../visitors/AbstractExprVisitor.java | 6 +- .../expression/visitors/AggregateChecker.java | 5 + .../expression/visitors/ConstantChecker.java | 5 + .../common/expression/visitors/ExprVisitor.java | 54 ++++---- .../visitors/ExpressionValidator.java | 5 + 25 files changed, 365 insertions(+), 195 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java index 1e71773..1a0b7d5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java @@ -31,6 +31,7 @@ import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; @@ -231,4 +232,9 @@ public class ConstantExpressionIdentifier implements ExprVisitor<Boolean, Identi public Boolean visitParameter(ValueExpressions.ParameterExpression e, IdentityHashMap<LogicalExpression, Object> value) throws RuntimeException { return false; } + + @Override + public Boolean visitTypedFieldExpr(TypedFieldExpr e, IdentityHashMap<LogicalExpression, Object> value) throws RuntimeException { + return false; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index 23df262..f1b50c9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -44,6 +44,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; @@ -78,7 +79,6 @@ import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder; import org.apache.drill.exec.expr.fn.DrillFuncHolder; import org.apache.drill.exec.expr.fn.ExceptionFunction; import org.apache.drill.exec.expr.fn.FunctionLookupContext; -import org.apache.drill.exec.expr.stat.TypedFieldExpr; import org.apache.drill.exec.record.TypedFieldId; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.resolver.FunctionResolver; @@ -323,7 +323,6 @@ public class ExpressionTreeMaterializer { } else { logger.warn("Unable to find value vector of path {}, returning null-int instance.", path); return new TypedFieldExpr(path, Types.OPTIONAL_INT); - // return NullExpression.INSTANCE; } } } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java index a58ce7c..bb8f3ac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java @@ -18,6 +18,8 @@ package org.apache.drill.exec.expr.stat; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.LogicalExpressionBase; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.visitors.ExprVisitor; import org.apache.parquet.column.statistics.Statistics; @@ -29,6 +31,7 @@ import java.util.List; * IS predicates for parquet filter pushdown. */ public class ParquetIsPredicates { + public static abstract class ParquetIsPredicate extends LogicalExpressionBase implements ParquetFilterPredicate { protected final LogicalExpression expr; @@ -54,12 +57,22 @@ public class ParquetIsPredicates { * IS NULL predicate. */ public static class IsNullPredicate extends ParquetIsPredicate { + private final boolean isArray; + public IsNullPredicate(LogicalExpression expr) { super(expr); + this.isArray = isArray(expr); } @Override public boolean canDrop(RangeExprEvaluator evaluator) { + + // for arrays we are not able to define exact number of nulls + // [1,2,3] vs [1,2] -> in second case 3 is absent and thus it's null but statistics shows no nulls + if (isArray) { + return false; + } + Statistics exprStat = expr.accept(evaluator, null); if (!ParquetPredicatesHelper.hasStats(exprStat)) { @@ -73,6 +86,16 @@ public class ParquetIsPredicates { return false; } } + + private boolean isArray(LogicalExpression expression) { + if (expression instanceof TypedFieldExpr) { + TypedFieldExpr typedFieldExpr = (TypedFieldExpr) expression; + SchemaPath schemaPath = typedFieldExpr.getPath(); + return schemaPath.isArray(); + } + return false; + } + } /** http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java index 2d241dc..2cc6a70 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java @@ -17,11 +17,11 @@ */ package org.apache.drill.exec.expr.stat; -import com.google.common.base.Preconditions; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.fn.CastFunctions; import org.apache.drill.common.expression.fn.FuncHolder; @@ -70,17 +70,20 @@ public class RangeExprEvaluator extends AbstractExprVisitor<Statistics, Void, Ru @Override public Statistics visitUnknown(LogicalExpression e, Void value) throws RuntimeException { - if (e instanceof TypedFieldExpr) { - TypedFieldExpr fieldExpr = (TypedFieldExpr) e; - final ColumnStatistics columnStatistics = columnStatMap.get(fieldExpr.getPath()); - if (columnStatistics != null) { - return columnStatistics.getStatistics(); - } else if (fieldExpr.getMajorType().equals(Types.OPTIONAL_INT)) { - // field does not exist. - IntStatistics intStatistics = new IntStatistics(); - intStatistics.setNumNulls(rowCount); // all values are nulls - return intStatistics; - } + // do nothing for the unknown expression + return null; + } + + @Override + public Statistics visitTypedFieldExpr(TypedFieldExpr typedFieldExpr, Void value) throws RuntimeException { + final ColumnStatistics columnStatistics = columnStatMap.get(typedFieldExpr.getPath()); + if (columnStatistics != null) { + return columnStatistics.getStatistics(); + } else if (typedFieldExpr.getMajorType().equals(Types.OPTIONAL_INT)) { + // field does not exist. + IntStatistics intStatistics = new IntStatistics(); + intStatistics.setNumNulls(rowCount); // all values are nulls + return intStatistics; } return null; } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/TypedFieldExpr.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/TypedFieldExpr.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/TypedFieldExpr.java deleted file mode 100644 index 4287929..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/TypedFieldExpr.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.expr.stat; - -import com.google.common.collect.Iterators; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.LogicalExpressionBase; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.expression.visitors.ExprVisitor; -import org.apache.drill.common.types.TypeProtos; - -import java.util.Iterator; - -public class TypedFieldExpr extends LogicalExpressionBase { - TypeProtos.MajorType type; - SchemaPath path; - - public TypedFieldExpr(SchemaPath path, TypeProtos.MajorType type) { - super(path.getPosition()); - this.path = path; - this.type = type; - } - - @Override - public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E { - return visitor.visitUnknown(this, value); - } - - @Override - public Iterator<LogicalExpression> iterator() { - return Iterators.emptyIterator(); - } - - @Override - public TypeProtos.MajorType getMajorType() { - return this.type; - } - - @Override - public String toString() { - return this.path.getRootSegment().getPath() + "(" + type.getMinorType() + "_" + type.getMode() +")"; - } - - public SchemaPath getPath() { - return this.path; - } - -} http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java index d5c8d94..dff83f6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.planner.common; import java.util.AbstractList; +import java.util.Collection; import java.util.List; import com.google.common.collect.Lists; @@ -178,22 +179,20 @@ public abstract class DrillRelOptUtil { } /** - * Travesal RexNode to find the item/flattern operator. Continue search if RexNode has a + * Travesal RexNode to find at least one operator in the given collection. Continue search if RexNode has a * RexInputRef which refers to a RexNode in project expressions. * * @param node : RexNode to search * @param projExprs : the list of project expressions. Empty list means there is No project operator underneath. + * @param operators collection of operators to find * @return : Return null if there is NONE; return the first appearance of item/flatten RexCall. */ - public static RexCall findItemOrFlatten( - final RexNode node, - final List<RexNode> projExprs) { + public static RexCall findOperators(final RexNode node, final List<RexNode> projExprs, final Collection<String> operators) { try { RexVisitor<Void> visitor = new RexVisitorImpl<Void>(true) { public Void visitCall(RexCall call) { - if ("item".equals(call.getOperator().getName().toLowerCase()) || - "flatten".equals(call.getOperator().getName().toLowerCase())) { + if (operators.contains(call.getOperator().getName().toLowerCase())) { throw new Util.FoundOne(call); /* throw exception to interrupt tree walk (this is similar to other utility methods in RexUtil.java */ } @@ -208,8 +207,7 @@ public abstract class DrillRelOptUtil { RexNode n = projExprs.get(index); if (n instanceof RexCall) { RexCall r = (RexCall) n; - if ("item".equals(r.getOperator().getName().toLowerCase()) || - "flatten".equals(r.getOperator().getName().toLowerCase())) { + if (operators.contains(r.getOperator().getName().toLowerCase())) { throw new Util.FoundOne(r); } } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java index d24abcc..7b978be 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java @@ -30,13 +30,23 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.util.Pair; import org.apache.drill.exec.planner.common.DrillRelOptUtil; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; public class DrillPushFilterPastProjectRule extends RelOptRule { public final static RelOptRule INSTANCE = new DrillPushFilterPastProjectRule(); - protected DrillPushFilterPastProjectRule() { + private static final Collection<String> BANNED_OPERATORS; + + static { + BANNED_OPERATORS = new ArrayList<>(2); + BANNED_OPERATORS.add("flatten"); + BANNED_OPERATORS.add("item"); + } + + private DrillPushFilterPastProjectRule() { super( operand( LogicalFilter.class, @@ -60,7 +70,7 @@ public class DrillPushFilterPastProjectRule extends RelOptRule { for (final RexNode pred : predList) { - if (DrillRelOptUtil.findItemOrFlatten(pred, projRel.getProjects()) == null) { + if (DrillRelOptUtil.findOperators(pred, projRel.getProjects(), BANNED_OPERATORS) == null) { qualifiedPredList.add(pred); } else { unqualifiedPredList.add(pred); http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java index a9e55dd..2d245a1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java @@ -19,8 +19,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.drill.common.expression.BooleanOperator; import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.PathSegment; -import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.fn.CastFunctions; import org.apache.drill.common.expression.fn.FuncHolder; @@ -41,7 +40,6 @@ import org.apache.drill.exec.expr.holders.ValueHolder; import org.apache.drill.exec.expr.stat.ParquetBooleanPredicates; import org.apache.drill.exec.expr.stat.ParquetComparisonPredicates; import org.apache.drill.exec.expr.stat.ParquetIsPredicates; -import org.apache.drill.exec.expr.stat.TypedFieldExpr; import org.apache.drill.exec.ops.UdfUtilities; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,11 +60,12 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression, /** * @param expr materialized filter expression * @param constantBoundaries set of constant expressions - * @param udfUtilities + * @param udfUtilities udf utilities + * + * @return logical expression */ public static LogicalExpression buildParquetFilterPredicate(LogicalExpression expr, final Set<LogicalExpression> constantBoundaries, UdfUtilities udfUtilities) { - final LogicalExpression predicate = expr.accept(new ParquetFilterBuilder(udfUtilities), constantBoundaries); - return predicate; + return expr.accept(new ParquetFilterBuilder(udfUtilities), constantBoundaries); } private ParquetFilterBuilder(UdfUtilities udfUtilities) { @@ -75,19 +74,16 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression, @Override public LogicalExpression visitUnknown(LogicalExpression e, Set<LogicalExpression> value) { - if (e instanceof TypedFieldExpr && - ! containsArraySeg(((TypedFieldExpr) e).getPath()) && - e.getMajorType().getMode() != TypeProtos.DataMode.REPEATED) { - // A filter is not qualified for push down, if - // 1. it contains an array segment : a.b[1], a.b[1].c.d - // 2. it's repeated type. - return e; - } - + // for the unknown expression, do nothing return null; } @Override + public LogicalExpression visitTypedFieldExpr(TypedFieldExpr typedFieldExpr, Set<LogicalExpression> value) throws RuntimeException { + return typedFieldExpr; + } + + @Override public LogicalExpression visitIntConstant(ValueExpressions.IntExpression intExpr, Set<LogicalExpression> value) throws RuntimeException { return intExpr; @@ -161,18 +157,6 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression, } } - private boolean containsArraySeg(final SchemaPath schemaPath) { - PathSegment seg = schemaPath.getRootSegment(); - - while (seg != null) { - if (seg.isArray()) { - return true; - } - seg = seg.getChild(); - } - return false; - } - private LogicalExpression getValueExpressionFromConst(ValueHolder holder, TypeProtos.MinorType type) { switch (type) { case INT: @@ -229,13 +213,9 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression, } if (CastFunctions.isCastFunction(funcName)) { - List<LogicalExpression> newArgs = new ArrayList(); - for (LogicalExpression arg : funcHolderExpr.args) { - final LogicalExpression newArg = arg.accept(this, value); - if (newArg == null) { - return null; - } - newArgs.add(newArg); + List<LogicalExpression> newArgs = generateNewExpressions(funcHolderExpr.args, value); + if (newArgs == null) { + return null; } return funcHolderExpr.copy(newArgs); @@ -244,15 +224,22 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression, } } - private LogicalExpression handleCompareFunction(FunctionHolderExpression functionHolderExpression, Set<LogicalExpression> value) { - List<LogicalExpression> newArgs = new ArrayList(); - - for (LogicalExpression arg : functionHolderExpression.args) { - LogicalExpression newArg = arg.accept(this, value); + private List<LogicalExpression> generateNewExpressions(List<LogicalExpression> expressions, Set<LogicalExpression> value) { + List<LogicalExpression> newExpressions = new ArrayList<>(); + for (LogicalExpression arg : expressions) { + final LogicalExpression newArg = arg.accept(this, value); if (newArg == null) { return null; } - newArgs.add(newArg); + newExpressions.add(newArg); + } + return newExpressions; + } + + private LogicalExpression handleCompareFunction(FunctionHolderExpression functionHolderExpression, Set<LogicalExpression> value) { + List<LogicalExpression> newArgs = generateNewExpressions(functionHolderExpression.args, value); + if (newArgs == null) { + return null; } String funcName = ((DrillSimpleFuncHolder) functionHolderExpression.getHolder()).getRegisteredNames()[0]; @@ -306,19 +293,6 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression, } } - private LogicalExpression handleCastFunction(FunctionHolderExpression functionHolderExpression, Set<LogicalExpression> value) { - for (LogicalExpression arg : functionHolderExpression.args) { - LogicalExpression newArg = arg.accept(this, value); - if (newArg == null) { - return null; - } - } - - String funcName = ((DrillSimpleFuncHolder) functionHolderExpression.getHolder()).getRegisteredNames()[0]; - - return null; - } - private static boolean isCompareFunction(String funcName) { return COMPARE_FUNCTIONS_SET.contains(funcName); } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java index 1ec10d8..4257150 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -39,6 +39,8 @@ import org.apache.drill.exec.planner.physical.ProjectPrel; import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.TimeUnit; @@ -46,6 +48,13 @@ public abstract class ParquetPushDownFilter extends StoragePluginOptimizerRule { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetPushDownFilter.class); + private static final Collection<String> BANNED_OPERATORS; + + static { + BANNED_OPERATORS = new ArrayList<>(1); + BANNED_OPERATORS.add("flatten"); + } + public static RelOptRule getFilterOnProject(OptimizerRulesContext optimizerRulesContext) { return new ParquetPushDownFilter( RelOptHelper.some(FilterPrel.class, RelOptHelper.some(ProjectPrel.class, RelOptHelper.any(ScanPrel.class))), @@ -127,7 +136,7 @@ public abstract class ParquetPushDownFilter extends StoragePluginOptimizerRule { final List<RexNode> qualifiedPredList = Lists.newArrayList(); for (final RexNode pred : predList) { - if (DrillRelOptUtil.findItemOrFlatten(pred, ImmutableList.<RexNode>of()) == null) { + if (DrillRelOptUtil.findOperators(pred, ImmutableList.of(), BANNED_OPERATORS) == null) { qualifiedPredList.add(pred); } } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java index 4991a22..a8c1218 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java @@ -41,11 +41,11 @@ import java.util.Set; import java.util.concurrent.TimeUnit; public class ParquetMetaStatCollector implements ColumnStatCollector{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetMetaStatCollector.class); + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetMetaStatCollector.class); - private final Metadata.ParquetTableMetadataBase parquetTableMetadata; - private final List<? extends Metadata.ColumnMetadata> columnMetadataList; - final Map<String, String> implicitColValues; + private final Metadata.ParquetTableMetadataBase parquetTableMetadata; + private final List<? extends Metadata.ColumnMetadata> columnMetadataList; + private final Map<String, String> implicitColValues; public ParquetMetaStatCollector(Metadata.ParquetTableMetadataBase parquetTableMetadata, List<? extends Metadata.ColumnMetadata> columnMetadataList, Map<String, String> implicitColValues) { @@ -82,11 +82,11 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{ columnMetadataMap.put(schemaPath, columnMetadata); } - for (final SchemaPath schemaPath : fields) { + for (final SchemaPath field : fields) { final PrimitiveType.PrimitiveTypeName primitiveType; final OriginalType originalType; - final Metadata.ColumnMetadata columnMetadata = columnMetadataMap.get(schemaPath); + final Metadata.ColumnMetadata columnMetadata = columnMetadataMap.get(field.getUnIndexed()); if (columnMetadata != null) { final Object min = columnMetadata.getMinValue(); @@ -95,7 +95,6 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{ primitiveType = this.parquetTableMetadata.getPrimitiveType(columnMetadata.getName()); originalType = this.parquetTableMetadata.getOriginalType(columnMetadata.getName()); - final Integer repetitionLevel = this.parquetTableMetadata.getRepetitionLevel(columnMetadata.getName()); int precision = 0; int scale = 0; // ColumnTypeMetadata_v3 stores information about scale and precision @@ -106,16 +105,16 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{ precision = columnTypeInfo.precision; } - statMap.put(schemaPath, getStat(min, max, numNull, primitiveType, originalType, repetitionLevel, scale, precision)); + statMap.put(field, getStat(min, max, numNull, primitiveType, originalType, scale, precision)); } else { - final String columnName = schemaPath.getRootSegment().getPath(); + final String columnName = field.getRootSegment().getPath(); if (implicitColValues.containsKey(columnName)) { TypeProtos.MajorType type = Types.required(TypeProtos.MinorType.VARCHAR); Statistics stat = new BinaryStatistics(); stat.setNumNulls(0); byte[] val = implicitColValues.get(columnName).getBytes(); stat.setMinMaxFromBytes(val, val); - statMap.put(schemaPath, new ColumnStatistics(stat, type)); + statMap.put(field, new ColumnStatistics(stat, type)); } } } @@ -128,7 +127,7 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{ } /** - * Builds column statistics using given primitiveType, originalType, repetitionLevel, scale, + * Builds column statistics using given primitiveType, originalType, scale, * precision, numNull, min and max values. * * @param min min value for statistics @@ -136,24 +135,18 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{ * @param numNull num_nulls for statistics * @param primitiveType type that determines statistics class * @param originalType type that determines statistics class - * @param repetitionLevel field repetition level * @param scale scale value (used for DECIMAL type) * @param precision precision value (used for DECIMAL type) * @return column statistics */ private ColumnStatistics getStat(Object min, Object max, Long numNull, PrimitiveType.PrimitiveTypeName primitiveType, OriginalType originalType, - Integer repetitionLevel, int scale, int precision) { + int scale, int precision) { Statistics stat = Statistics.getStatsBasedOnType(primitiveType); Statistics convertedStat = stat; TypeProtos.MajorType type = ParquetGroupScan.getType(primitiveType, originalType, scale, precision); - // Change to repeated if repetitionLevel > 0 - if (repetitionLevel != null && repetitionLevel > 0) { - type = Types.withScaleAndPrecision(type.getMinorType(), TypeProtos.DataMode.REPEATED, scale, precision); - } - if (numNull != null) { stat.setNumNulls(numNull); } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDownForComplexTypes.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDownForComplexTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDownForComplexTypes.java new file mode 100644 index 0000000..5cbe5cf --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDownForComplexTypes.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.parquet; + +import org.apache.drill.PlanTestBase; +import org.apache.drill.exec.util.StoragePluginTestUtils; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; + +public class TestParquetFilterPushDownForComplexTypes extends PlanTestBase { + + private static final String TABLE_PATH = "parquet/users"; + private static final String TABLE_NAME = String.format("%s.`%s`", StoragePluginTestUtils.DFS_PLUGIN_NAME, TABLE_PATH); + + @BeforeClass + public static void copyData() { + /* + Parquet schema: + message complex_users { + required group user { + required int32 id; + optional int32 age; + repeated int32 hobby_ids; + optional boolean active; + } + } + + Data set: + users_1.parquet + {"id":1,"age":25,"hobby_ids":[1,2,3],"active":true} + {"id":2,"age":28,"hobby_ids":[1,2,5],"active":true} + + users_2.parquet + {"id":3,"age":31,"hobby_ids":[1,2,3],"active":true} + {"id":4,"age":32,"hobby_ids":[4,10,18],"active":false} + + users_3.parquet + {"id":5,"hobby_ids":[11,12,13,14,15]} + + users_4.parquet + {"id":6,"age":41,"hobby_ids":[20,21,22],"active":true} + {"id":7,"hobby_ids":[20,21,22,24]} + + users_5.parquet + {"id":8,"age":41,"hobby_ids":[],"active":false} + + users_6.parquet + {"id":9,"age":20,"hobby_ids":[],"active":false} + {"id":10,"age":21,"hobby_ids":[26,28,29]} + + users_7.parquet + {"id":11,"age":23,"hobby_ids":[10,11,12],"active":true} + {"id":12,"age":35,"hobby_ids":[22,23,24],"active":false} + {"id":13,"age":25,"hobby_ids":[14,22,26]} + + */ + dirTestWatcher.copyResourceToRoot(Paths.get(TABLE_PATH)); + } + + @Test + public void testPushDownArray() throws Exception { + testParquetFilterPushDown("t.`user`.hobby_ids[0] = 1", 3, 2); + testParquetFilterPushDown("t.`user`.hobby_ids[0] = 100", 0, 1); + testParquetFilterPushDown("t.`user`.hobby_ids[0] <> 1", 8, 6); + testParquetFilterPushDown("t.`user`.hobby_ids[2] > 20", 5, 3); + testParquetFilterPushDown("t.`user`.hobby_ids[0] between 10 and 20", 5, 4); + testParquetFilterPushDown("t.`user`.hobby_ids[4] = 15", 1, 3); + testParquetFilterPushDown("t.`user`.hobby_ids[2] is not null", 11, 6); + testParquetFilterPushDown("t.`user`.hobby_ids[3] is null", 11, 7); + } + + @Test + public void testPushDownComplexIntColumn() throws Exception { + testParquetFilterPushDown("t.`user`.age = 31", 1, 2); + testParquetFilterPushDown("t.`user`.age = 1", 0, 1); + testParquetFilterPushDown("t.`user`.age <> 20", 10, 6); + testParquetFilterPushDown("t.`user`.age > 30", 5, 4); + testParquetFilterPushDown("t.`user`.age between 20 and 30", 6, 3); + testParquetFilterPushDown("t.`user`.age is not null", 11, 6); + testParquetFilterPushDown("t.`user`.age is null", 2, 2); + } + + @Test + public void testPushDownComplexBooleanColumn() throws Exception { + testParquetFilterPushDown("t.`user`.active is true", 5, 4); + testParquetFilterPushDown("t.`user`.active is not true", 8, 6); + testParquetFilterPushDown("t.`user`.active is false", 4, 4); + testParquetFilterPushDown("t.`user`.active is not false", 9, 6); + testParquetFilterPushDown("t.`user`.active is not null", 9, 6); + testParquetFilterPushDown("t.`user`.active is null", 4, 4); + } + + + private void testParquetFilterPushDown(String predicate, int expectedRowCount, int expectRowGroupsNumber) throws Exception { + String query = String.format("select * from %s t where %s", TABLE_NAME, predicate); + + int actualRowCount = testSql(query); + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + String expectRowGroupsNumberPattern = "numRowGroups=" + expectRowGroupsNumber; + testPlanMatchingPatterns(query, new String[] {expectRowGroupsNumberPattern}, new String[] {}); + } + +} + http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/users/users_1.parquet b/exec/java-exec/src/test/resources/parquet/users/users_1.parquet new file mode 100644 index 0000000..9d9d511 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/users/users_2.parquet b/exec/java-exec/src/test/resources/parquet/users/users_2.parquet new file mode 100644 index 0000000..f48c43a Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/users/users_3.parquet b/exec/java-exec/src/test/resources/parquet/users/users_3.parquet new file mode 100644 index 0000000..1c1aa8e Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/users/users_4.parquet b/exec/java-exec/src/test/resources/parquet/users/users_4.parquet new file mode 100644 index 0000000..66a7288 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/users/users_5.parquet b/exec/java-exec/src/test/resources/parquet/users/users_5.parquet new file mode 100644 index 0000000..01aafd1 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/users/users_6.parquet b/exec/java-exec/src/test/resources/parquet/users/users_6.parquet new file mode 100644 index 0000000..c12105a Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_6.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_7.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/users/users_7.parquet b/exec/java-exec/src/test/resources/parquet/users/users_7.parquet new file mode 100644 index 0000000..00f8016 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_7.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java index 583046a..9ea57b1 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java +++ b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java @@ -90,19 +90,25 @@ public class SchemaPath extends LogicalExpressionBase { } public NamePart getAsNamePart() { - return getNamePart(rootSegment); + return getNamePart(rootSegment, false); } - private static NamePart getNamePart(PathSegment s) { + private static NamePart getNamePart(PathSegment s, boolean skipArraySegment) { if (s == null) { return null; } NamePart.Builder b = NamePart.newBuilder(); if (s.getChild() != null) { - b.setChild(getNamePart(s.getChild())); + NamePart namePart = getNamePart(s.getChild(), skipArraySegment); + if (namePart != null) { + b.setChild(namePart); + } } if (s.isArray()) { + if (skipArraySegment) { + return null; + } if (s.getArraySegment().hasIndex()) { throw new IllegalStateException("You cannot convert a indexed schema path to a NamePart. NameParts can only reference Vectors, not individual records or values."); } @@ -129,6 +135,18 @@ public class SchemaPath extends LogicalExpressionBase { } /** + * Returns schema path with for arrays without index. + * Is used to find column statistics in parquet metadata. + * Example: a.b.c[0] -> a.b.c + * + * @return un-indexed schema path + */ + public SchemaPath getUnIndexed() { + NamePart namePart = getNamePart(rootSegment, true); + return create(namePart); + } + + /** * Parses input string using the same rules which are used for the field in the query. * If a string contains dot outside back-ticks, or there are no backticks in the string, * will be created {@link SchemaPath} with the {@link NameSegment} @@ -255,11 +273,6 @@ public class SchemaPath extends LogicalExpressionBase { return new SchemaPath(newRoot); } - public SchemaPath getUnindexedArrayChild() { - NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(null)); - return new SchemaPath(newRoot); - } - public SchemaPath getChild(int index) { NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(index)); return new SchemaPath(newRoot); http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java b/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java new file mode 100644 index 0000000..93c7a3c --- /dev/null +++ b/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.expression; + +import org.apache.drill.common.expression.visitors.ExprVisitor; +import org.apache.drill.common.types.TypeProtos; + +import java.util.Collections; +import java.util.Iterator; + +public class TypedFieldExpr extends LogicalExpressionBase { + private final TypeProtos.MajorType type; + private final SchemaPath path; + + public TypedFieldExpr(SchemaPath path, TypeProtos.MajorType type) { + super(path.getPosition()); + this.path = path; + this.type = type; + } + + @Override + public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E { + return visitor.visitTypedFieldExpr(this, value); + } + + @Override + public Iterator<LogicalExpression> iterator() { + return Collections.emptyIterator(); + } + + @Override + public TypeProtos.MajorType getMajorType() { + return this.type; + } + + @Override + public String toString() { + return this.path.getRootSegment().getPath() + "(" + type.getMinorType() + "_" + type.getMode() +")"; + } + + public SchemaPath getPath() { + return this.path; + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java index 189e33d..5356813 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java +++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java @@ -26,6 +26,7 @@ import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; import org.apache.drill.common.expression.ValueExpressions.DateExpression; @@ -45,7 +46,6 @@ import org.apache.drill.common.expression.ValueExpressions.TimeExpression; import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> implements ExprVisitor<T, VAL, EXCEP> { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractExprVisitor.class); @Override public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP { @@ -178,4 +178,8 @@ public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> imple return visitUnknown(e, value); } + @Override + public T visitTypedFieldExpr(TypedFieldExpr e, VAL value) throws EXCEP { + return visitUnknown(e, value); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java index 9a3cdcc..f6fe89d 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java +++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java @@ -28,6 +28,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; @@ -209,4 +210,8 @@ public final class AggregateChecker implements ExprVisitor<Boolean, ErrorCollect return false; } + @Override + public Boolean visitTypedFieldExpr(TypedFieldExpr e, ErrorCollector value) throws RuntimeException { + return false; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java index 67fe12f..0468bc2 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java +++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java @@ -28,6 +28,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; @@ -213,4 +214,8 @@ final class ConstantChecker implements ExprVisitor<Boolean, ErrorCollector, Runt return false; } + @Override + public Boolean visitTypedFieldExpr(TypedFieldExpr e, ErrorCollector value) throws RuntimeException { + return false; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java index 7c59f3c..e6198ae 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java +++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java @@ -26,6 +26,7 @@ import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; import org.apache.drill.common.expression.ValueExpressions.DateExpression; @@ -45,30 +46,31 @@ import org.apache.drill.common.expression.ValueExpressions.TimeExpression; import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; public interface ExprVisitor<T, VAL, EXCEP extends Exception> { - public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP; - public T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP; - public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP; - public T visitBooleanOperator(BooleanOperator call, VAL value) throws EXCEP; - public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP; - public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP; - public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP; - public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP; - public T visitDateConstant(DateExpression intExpr, VAL value) throws EXCEP; - public T visitTimeConstant(TimeExpression intExpr, VAL value) throws EXCEP; - public T visitTimeStampConstant(TimeStampExpression intExpr, VAL value) throws EXCEP; - public T visitIntervalYearConstant(IntervalYearExpression intExpr, VAL value) throws EXCEP; - public T visitIntervalDayConstant(IntervalDayExpression intExpr, VAL value) throws EXCEP; - public T visitDecimal9Constant(Decimal9Expression decExpr, VAL value) throws EXCEP; - public T visitDecimal18Constant(Decimal18Expression decExpr, VAL value) throws EXCEP; - public T visitDecimal28Constant(Decimal28Expression decExpr, VAL value) throws EXCEP; - public T visitDecimal38Constant(Decimal38Expression decExpr, VAL value) throws EXCEP; - public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP; - public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP; - public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP; - public T visitNullConstant(TypedNullConstant e, VAL value) throws EXCEP; - public T visitNullExpression(NullExpression e, VAL value) throws EXCEP; - public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP; - public T visitCastExpression(CastExpression e, VAL value) throws EXCEP; - public T visitConvertExpression(ConvertExpression e, VAL value) throws EXCEP; - public T visitParameter(ParameterExpression e, VAL value) throws EXCEP; + T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP; + T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP; + T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP; + T visitBooleanOperator(BooleanOperator call, VAL value) throws EXCEP; + T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP; + T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP; + T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP; + T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP; + T visitDateConstant(DateExpression intExpr, VAL value) throws EXCEP; + T visitTimeConstant(TimeExpression intExpr, VAL value) throws EXCEP; + T visitTimeStampConstant(TimeStampExpression intExpr, VAL value) throws EXCEP; + T visitIntervalYearConstant(IntervalYearExpression intExpr, VAL value) throws EXCEP; + T visitIntervalDayConstant(IntervalDayExpression intExpr, VAL value) throws EXCEP; + T visitDecimal9Constant(Decimal9Expression decExpr, VAL value) throws EXCEP; + T visitDecimal18Constant(Decimal18Expression decExpr, VAL value) throws EXCEP; + T visitDecimal28Constant(Decimal28Expression decExpr, VAL value) throws EXCEP; + T visitDecimal38Constant(Decimal38Expression decExpr, VAL value) throws EXCEP; + T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP; + T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP; + T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP; + T visitNullConstant(TypedNullConstant e, VAL value) throws EXCEP; + T visitNullExpression(NullExpression e, VAL value) throws EXCEP; + T visitUnknown(LogicalExpression e, VAL value) throws EXCEP; + T visitCastExpression(CastExpression e, VAL value) throws EXCEP; + T visitConvertExpression(ConvertExpression e, VAL value) throws EXCEP; + T visitParameter(ParameterExpression e, VAL value) throws EXCEP; + T visitTypedFieldExpr(TypedFieldExpr e, VAL value) throws EXCEP; } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java index e8cbc2b..c32825a 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java +++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java @@ -28,6 +28,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; @@ -236,4 +237,8 @@ public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, Ru return null; } + @Override + public Void visitTypedFieldExpr(TypedFieldExpr e, ErrorCollector value) throws RuntimeException { + return null; + } }
