IMPALA-4592: Improve error msg for non-deterministic predicates. Impala cannot correctly evaluate or assign some non-deterministic predicates. This patch improves the error message shown when trying to evaluate such unsupported predicates for the purpose of partition pruning.
Change-Id: I94765f62bde94f4faa7fc5c26d928099ca1496d1 Reviewed-on: http://gerrit.cloudera.org:8080/5386 Reviewed-by: Alex Behm <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/6098ac71 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/6098ac71 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/6098ac71 Branch: refs/heads/master Commit: 6098ac7162742c11350de708188ce6c3f7ce11a7 Parents: 5188f87 Author: Alex Behm <[email protected]> Authored: Tue Dec 6 14:30:43 2016 -0800 Committer: Internal Jenkins <[email protected]> Committed: Wed Dec 7 06:27:51 2016 +0000 ---------------------------------------------------------------------- .../org/apache/impala/analysis/PartitionSet.java | 5 +++-- .../impala/planner/HdfsPartitionFilter.java | 18 +++++++++++------- .../impala/planner/HdfsPartitionPruner.java | 6 +++--- .../queries/PlannerTest/hdfs.test | 11 +++++++++++ 4 files changed, 28 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java index 6de2d93..3ba2ad2 100644 --- a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java +++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java @@ -25,7 +25,7 @@ import org.apache.impala.catalog.Column; import org.apache.impala.catalog.HdfsPartition; import org.apache.impala.catalog.Table; import org.apache.impala.common.AnalysisException; -import org.apache.impala.common.InternalException; +import org.apache.impala.common.ImpalaException; import org.apache.impala.common.Reference; import org.apache.impala.planner.HdfsPartitionPruner; import org.apache.impala.thrift.TPartitionKeyValue; @@ -87,7 +87,8 @@ public class PartitionSet extends PartitionSpecBase { try { HdfsPartitionPruner pruner = new HdfsPartitionPruner(desc); partitions_ = pruner.prunePartitions(analyzer, transformedConjuncts, true); - } catch (InternalException e) { + } catch (ImpalaException e) { + if (e instanceof AnalysisException) throw (AnalysisException) e; throw new AnalysisException("Partition expr evaluation failed in the backend.", e); } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java index 7368358..3c0fb15 100644 --- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java +++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java @@ -21,9 +21,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.impala.analysis.Analyzer; import org.apache.impala.analysis.Expr; import org.apache.impala.analysis.ExprSubstitutionMap; @@ -33,10 +30,14 @@ import org.apache.impala.analysis.SlotRef; import org.apache.impala.catalog.Column; import org.apache.impala.catalog.HdfsPartition; import org.apache.impala.catalog.HdfsTable; -import org.apache.impala.common.InternalException; +import org.apache.impala.common.ImpalaException; +import org.apache.impala.common.NotImplementedException; import org.apache.impala.service.FeSupport; import org.apache.impala.thrift.TColumnValue; import org.apache.impala.thrift.TResultRow; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -83,7 +84,7 @@ public class HdfsPartitionFilter { * that pass the filter. */ public HashSet<Long> getMatchingPartitionIds(ArrayList<HdfsPartition> partitions, - Analyzer analyzer) throws InternalException { + Analyzer analyzer) throws ImpalaException { HashSet<Long> result = new HashSet<Long>(); // List of predicates to evaluate ArrayList<Expr> predicates = new ArrayList<Expr>(partitions.size()); @@ -110,7 +111,7 @@ public class HdfsPartitionFilter { * for the partition cols with the respective partition-key values. */ private Expr buildPartitionPredicate(HdfsPartition partition, Analyzer analyzer) - throws InternalException { + throws ImpalaException { // construct smap ExprSubstitutionMap sMap = new ExprSubstitutionMap(); for (int i = 0; i < refdKeys_.size(); ++i) { @@ -123,7 +124,10 @@ public class HdfsPartitionFilter { LOG.trace("buildPartitionPredicate: " + literalPredicate.toSql() + " " + literalPredicate.debugString()); } - Preconditions.checkState(literalPredicate.isConstant()); + if (!literalPredicate.isConstant()) { + throw new NotImplementedException( + "Unsupported non-deterministic predicate: " + predicate_.toSql()); + } return literalPredicate; } } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java index b8eb7ed..046240f 100644 --- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java +++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java @@ -42,7 +42,7 @@ import org.apache.impala.analysis.TupleDescriptor; import org.apache.impala.catalog.HdfsPartition; import org.apache.impala.catalog.HdfsTable; import org.apache.impala.common.AnalysisException; -import org.apache.impala.common.InternalException; +import org.apache.impala.common.ImpalaException; import org.apache.impala.rewrite.BetweenToCompoundRule; import org.apache.impala.rewrite.ExprRewriter; import org.slf4j.Logger; @@ -94,7 +94,7 @@ public class HdfsPartitionPruner { */ public List<HdfsPartition> prunePartitions( Analyzer analyzer, List<Expr> conjuncts, boolean allowEmpty) - throws InternalException, AnalysisException { + throws ImpalaException { // Start with creating a collection of partition filters for the applicable conjuncts. List<HdfsPartitionFilter> partitionFilters = Lists.newArrayList(); // Conjuncts that can be evaluated from the partition key values. @@ -438,7 +438,7 @@ public class HdfsPartitionPruner { * filters that could not be evaluated from the partition key values. */ private void evalPartitionFiltersInBe(List<HdfsPartitionFilter> filters, - HashSet<Long> matchingPartitionIds, Analyzer analyzer) throws InternalException { + HashSet<Long> matchingPartitionIds, Analyzer analyzer) throws ImpalaException { Map<Long, HdfsPartition> partitionMap = tbl_.getPartitionMap(); // Set of partition ids that pass a filter HashSet<Long> matchingIds = Sets.newHashSet(); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test index 606425b..83f1b46 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test @@ -1019,3 +1019,14 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional.alltypes] partitions=0/24 files=0 size=0B ==== +# IMPALA-4592: Test that we bail on evaluating non-deterministic predicates when trying +# to prune partitions. +select * from functional.alltypes where rand() > 100 +---- PLAN +not implemented: Unsupported non-deterministic predicate: rand() > 100 +==== +# IMPALA-4592: Same as above but the predicate references a partition column. +select * from functional.alltypes where rand() > year +---- PLAN +not implemented: Unsupported non-deterministic predicate: rand() > year +====
