This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit e473f034dc0195f4011165177c913c3a238d2ff7 Author: Steve Carlin <[email protected]> AuthorDate: Mon Jan 27 15:40:22 2025 -0800 IMPALA-13042: Calcite Planner; Enable partition pruning Enables partition pruning in the HdfsScan RelNode. The PrunedPartitionHelper is used to separate the conjuncts and is a wrapper around the HdfsPartitionPruner. There are tests that currently exist in the Impala test framework that check the runtime profile for the number of files read from the table scan which are fixed with this commit. One small modification was made to the "preserveRootTypes" parameter in HdfsPartitionPruner. When it was set to false, the Calcite planner failed in one place. It makes sense for the main code line that the root type should not change, and this was tested on a full Jenkins run. Change-Id: I8c698b857555baeae347835b4a6b39d035f12405 Reviewed-on: http://gerrit.cloudera.org:8080/22409 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Steve Carlin <[email protected]> --- .../apache/impala/planner/HdfsPartitionFilter.java | 2 +- .../apache/impala/planner/HdfsPartitionPruner.java | 9 +- .../impala/calcite/rel/node/ImpalaHdfsScanRel.java | 13 +-- .../calcite/rel/util/PrunedPartitionHelper.java | 98 ++++++++++++++++++++++ 4 files changed, 113 insertions(+), 9 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java index d2189a26d..dbe63a03c 100644 --- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java +++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java @@ -120,7 +120,7 @@ public class HdfsPartitionFilter { lhsSlotRefs_.get(i), p.getPartitionValues().get(refdKeys_.get(i))); } - Expr literalPredicate = predicate_.substitute(sMap, analyzer, false); + Expr literalPredicate = predicate_.substitute(sMap, analyzer, true); if (LOG.isTraceEnabled()) { LOG.trace("buildPartitionPredicate: " + literalPredicate.toSql() + " " + literalPredicate.debugString()); diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java index fa0053be0..56a7e6dba 100644 --- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java +++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java @@ -129,8 +129,7 @@ public class HdfsPartitionPruner { Iterator<Expr> it = conjuncts.iterator(); while (it.hasNext()) { Expr conjunct = it.next(); - if (conjunct.isBoundBySlotIds(partitionSlots_) && - !conjunct.contains(Expr.IS_NONDETERMINISTIC_BUILTIN_FN_PREDICATE)) { + if (isPartitionPrunedFilterConjunct(partitionSlots_, conjunct)) { // Check if the conjunct can be evaluated from the partition metadata. // Use a cloned conjunct to rewrite BetweenPredicates and allow // canEvalUsingPartitionMd() to fold constant expressions without modifying @@ -189,6 +188,12 @@ public class HdfsPartitionPruner { return new Pair<>(results, partitionConjuncts); } + public static boolean isPartitionPrunedFilterConjunct(List<SlotId> partitionSlots, + Expr conjunct) { + return conjunct.isBoundBySlotIds(partitionSlots) && + !conjunct.contains(Expr.IS_NONDETERMINISTIC_BUILTIN_FN_PREDICATE); + } + /** * Prune partitions based on eligibility of simple limit optimization: * - Either the table ref should not already have a TABLESAMPLE clause diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java index 18be43898..4761b5ea4 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java @@ -27,6 +27,7 @@ import org.apache.impala.analysis.SlotRef; import org.apache.impala.analysis.TupleDescriptor; import org.apache.impala.calcite.rel.phys.ImpalaHdfsScanNode; import org.apache.impala.calcite.rel.util.ExprConjunctsConverter; +import org.apache.impala.calcite.rel.util.PrunedPartitionHelper; import org.apache.impala.calcite.schema.CalciteTable; import org.apache.impala.calcite.util.SimplifiedAnalyzer; import org.apache.impala.catalog.FeFsPartition; @@ -70,13 +71,13 @@ public class ImpalaHdfsScanRel extends TableScan ExprConjunctsConverter converter = new ExprConjunctsConverter( context.filterCondition_, outputExprs, getCluster().getRexBuilder(), context.ctx_.getRootAnalyzer()); - List<? extends FeFsPartition> impalaPartitions = table.getPrunedPartitions( - context.ctx_.getRootAnalyzer(), tupleDesc); - // TODO: All conjuncts will be nonpartitioned conjuncts until the partition - // pruning feature is committed. - List<Expr> filterConjuncts = converter.getImpalaConjuncts(); - List<Expr> partitionConjuncts = new ArrayList<>(); + PrunedPartitionHelper pph = new PrunedPartitionHelper(table, converter, + tupleDesc, getCluster().getRexBuilder(), context.ctx_.getRootAnalyzer()); + + List<? extends FeFsPartition> impalaPartitions = pph.getPrunedPartitions(); + List<Expr> partitionConjuncts = pph.getPartitionedConjuncts(); + List<Expr> filterConjuncts = pph.getNonPartitionedConjuncts(); PlanNodeId nodeId = context.ctx_.getNextNodeId(); diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/PrunedPartitionHelper.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/PrunedPartitionHelper.java new file mode 100644 index 000000000..8d699efae --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/PrunedPartitionHelper.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.impala.calcite.rel.util; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.impala.analysis.Analyzer; +import org.apache.impala.analysis.Expr; +import org.apache.impala.analysis.SlotId; +import org.apache.impala.analysis.TupleDescriptor; +import org.apache.impala.calcite.schema.CalciteTable; +import org.apache.impala.catalog.FeFsPartition; +import org.apache.impala.common.ImpalaException; +import org.apache.impala.common.Pair; +import org.apache.impala.planner.HdfsPartitionPruner; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * PrunedPartitionHelper is a wrapper around the Impala Planner's partition + * pruner and holds the partitions remaining after pruning and the conjuncts + * separated by the ones that pruned and the ones that did not prune. + */ +public class PrunedPartitionHelper { + + private final List<? extends FeFsPartition> prunedPartitions_; + + private final List<Expr> partitionedConjuncts_; + + private final List<Expr> nonPartitionedConjuncts_; + + public PrunedPartitionHelper(CalciteTable table, + ExprConjunctsConverter converter, TupleDescriptor tupleDesc, + RexBuilder rexBuilder, + Analyzer analyzer) throws ImpalaException { + + HdfsPartitionPruner pruner = new HdfsPartitionPruner(tupleDesc); + + List<Expr> conjuncts = converter.getImpalaConjuncts(); + // IMPALA-13849: tblref is null. Tablesampling is disabled. + Pair<List<? extends FeFsPartition>, List<Expr>> impalaPair = + pruner.prunePartitions(analyzer, new ArrayList<>(conjuncts), true, + null); + + prunedPartitions_ = impalaPair.first; + + ImmutableList.Builder<Expr> partitionedConjBuilder = + new ImmutableList.Builder(); + ImmutableList.Builder<Expr> nonPartitionedConjBuilder = + new ImmutableList.Builder(); + List<SlotId> partitionSlots = tupleDesc.getPartitionSlots(); + + for (Expr conjunct : conjuncts) { + if (HdfsPartitionPruner.isPartitionPrunedFilterConjunct(partitionSlots, conjunct)) { + partitionedConjBuilder.add(conjunct); + } else { + nonPartitionedConjBuilder.add(conjunct); + } + } + + partitionedConjuncts_ = partitionedConjBuilder.build(); + nonPartitionedConjuncts_ = nonPartitionedConjBuilder.build(); + } + + public List<? extends FeFsPartition> getPrunedPartitions() { + return prunedPartitions_; + } + + public List<Expr> getPartitionedConjuncts() { + return partitionedConjuncts_; + } + + public List<Expr> getNonPartitionedConjuncts() { + return nonPartitionedConjuncts_; + } +}
