This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 314e67fbafa HIVE-29380: Drop synthetic HiveProject and
HiveFilterProjectTSTransposeRule (#6241)
314e67fbafa is described below
commit 314e67fbafae7fa8728d5918af79c7d062204aef
Author: Stamatis Zampetakis <[email protected]>
AuthorDate: Fri Jan 2 20:11:44 2026 +0100
HIVE-29380: Drop synthetic HiveProject and HiveFilterProjectTSTransposeRule
(#6241)
---
.../calcite/reloperators/HiveProject.java | 27 +----
.../calcite/reloperators/HiveTableScan.java | 7 +-
.../rules/HiveFilterProjectTSTransposeRule.java | 134 ---------------------
.../rules/HiveFilterProjectTransposeRule.java | 41 +++++--
.../calcite/rules/HiveRelFieldTrimmer.java | 6 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 7 +-
6 files changed, 34 insertions(+), 188 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
index 46edad00f7a..fa0bbb307a1 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
@@ -26,12 +26,10 @@
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelShuttle;
-import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.util.Util;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
@@ -46,8 +44,6 @@ public class HiveProject extends Project implements
HiveRelNode {
// Information about correlations within a subquery.
private final CorrelationInfoSupplier correlationInfos;
- private boolean isSysnthetic;
-
/**
* Creates a HiveProject.
* @param cluster
@@ -110,22 +106,7 @@ public static HiveProject create(RelOptCluster cluster,
RelNode child, List<? ex
@Override
public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> exps,
RelDataType rowType) {
assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION);
- HiveProject hp = new HiveProject(getCluster(), traitSet, input, exps,
rowType);
- if (this.isSynthetic()) {
- hp.setSynthetic();
- }
-
- return hp;
- }
-
- // TODO: this should come through RelBuilder to the constructor as opposed to
- // set method. This requires calcite change
- public void setSynthetic() {
- this.isSysnthetic = true;
- }
-
- public boolean isSynthetic() {
- return isSysnthetic;
+ return new HiveProject(getCluster(), traitSet, input, exps, rowType);
}
//required for HiveRelDecorrelator
@@ -136,12 +117,6 @@ public boolean isSynthetic() {
return shuttle.visit(this);
}
- @Override
- public RelWriter explainTerms(RelWriter pw) {
- return super.explainTerms(pw)
- .itemIf("synthetic", this.isSysnthetic, pw.getDetailLevel() ==
SqlExplainLevel.DIGEST_ATTRIBUTES);
- }
-
public List<HiveCorrelationInfo> getCorrelationInfos() {
return correlationInfos.get();
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
index 2c3e4c6e6e3..05e8b448f62 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
@@ -261,13 +261,8 @@ public RelNode project(ImmutableBitSet fieldsUsed,
Set<RelDataTypeField> extraFi
fieldNames));
// 5. Add Proj on top of TS
- HiveProject hp = (HiveProject) relBuilder.push(newHT)
+ return relBuilder.push(newHT)
.project(exprList, new ArrayList<String>(fieldNames)).build();
-
- // 6. Set synthetic flag, so that we would push filter below this one
- hp.setSynthetic();
-
- return hp;
}
public List<Integer> getNeededColIndxsFrmReloptHT() {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
deleted file mode 100644
index 6eace95a4a2..00000000000
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
-
-import org.apache.calcite.adapter.druid.DruidQuery;
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.core.RelFactories.FilterFactory;
-import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.rules.ProjectMergeRule;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexOver;
-import org.apache.calcite.rex.RexUtil;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
-import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
-
-import java.util.Collections;
-
-//TODO: Remove this once Calcite FilterProjectTransposeRule can take rule
operand
-public class HiveFilterProjectTSTransposeRule extends RelOptRule {
-
- public final static HiveFilterProjectTSTransposeRule INSTANCE =
- new HiveFilterProjectTSTransposeRule(
- Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
HiveProject.class,
- HiveRelFactories.HIVE_PROJECT_FACTORY, TableScan.class,
ProjectMergeRule.DEFAULT_BLOAT);
-
- public final static HiveFilterProjectTSTransposeRule INSTANCE_DRUID =
- new HiveFilterProjectTSTransposeRule(
- Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
HiveProject.class,
- HiveRelFactories.HIVE_PROJECT_FACTORY, DruidQuery.class,
ProjectMergeRule.DEFAULT_BLOAT);
-
- private final RelFactories.FilterFactory filterFactory;
- private final RelFactories.ProjectFactory projectFactory;
- private final int bloat;
-
- private HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass,
- FilterFactory filterFactory, Class<? extends Project> projectClass,
- ProjectFactory projectFactory, Class<? extends RelNode> tsClass,
- int bloat) {
- super(operand(filterClass, operand(projectClass, operand(tsClass,
none()))));
- this.filterFactory = filterFactory;
- this.projectFactory = projectFactory;
- this.bloat = bloat;
- }
-
- @Override
- public boolean matches(RelOptRuleCall call) {
- final HiveProject projRel = call.rel(1);
-
- // Assumption:
- // 1. This will be run last after PP, Col Pruning in the PreJoinOrder
- // optimizations.
- // 2. If ProjectRel is not synthetic then PPD would have already pushed
- // relevant pieces down and hence no point in running PPD again.
- // 3. For synthetic Projects we don't care about non deterministic UDFs
- if (!projRel.isSynthetic()) {
- return false;
- }
-
- return super.matches(call);
- }
-
- // ~ Methods ----------------------------------------------------------------
-
- // implement RelOptRule
- public void onMatch(RelOptRuleCall call) {
- final Filter filter = call.rel(0);
- final Project project = call.rel(1);
-
- if (RexOver.containsOver(project.getProjects(), null)) {
- // In general a filter cannot be pushed below a windowing calculation.
- // Applying the filter before the aggregation function changes
- // the results of the windowing invocation.
- //
- // When the filter is on the PARTITION BY expression of the OVER clause
- // it can be pushed down. For now we don't support this.
- return;
- }
-
- if (RexUtil.containsCorrelation(filter.getCondition())) {
- // If there is a correlation condition anywhere in the filter, don't
- // push this filter past project since in some cases it can prevent a
- // Correlate from being de-correlated.
- return;
- }
-
- // convert the filter to one that references the child of the project
- RexNode newCondition =
HiveRelOptUtil.pushPastProjectUnlessBloat(filter.getCondition(), project,
bloat);
- if (newCondition == null) {
- return;
- }
-
- // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
- // nullable and not-nullable conditions, but a CAST might get in the way of
- // other rewrites.
- final RelDataTypeFactory typeFactory =
filter.getCluster().getTypeFactory();
- if (RexUtil.isNullabilityCast(typeFactory, newCondition)) {
- newCondition = ((RexCall) newCondition).getOperands().get(0);
- }
-
- RelNode newFilterRel = filterFactory == null ?
filter.copy(filter.getTraitSet(),
- project.getInput(), newCondition) :
filterFactory.createFilter(project.getInput(),
- newCondition);
-
- RelNode newProjRel = projectFactory == null ?
project.copy(project.getTraitSet(), newFilterRel,
- project.getProjects(), project.getRowType()) :
projectFactory.createProject(newFilterRel,
- Collections.emptyList(), project.getProjects(),
project.getRowType().getFieldNames());
-
- call.transformTo(newProjRel);
- }
-}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
index 819231cb54d..5db95b0ca48 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
@@ -23,7 +23,9 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Predicate;
+import org.apache.calcite.adapter.druid.DruidQuery;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRuleCall;
@@ -35,6 +37,7 @@
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
import org.apache.calcite.rel.rules.ProjectMergeRule;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -59,14 +62,22 @@
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
-import
org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil.RewritablePKFKJoinInfo;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
public class HiveFilterProjectTransposeRule extends FilterProjectTransposeRule
{
+ private static RelOptRuleOperand joinOperand() {
+ Predicate<Join> isRewritable = join -> {
+ RelMetadataQuery mq = join.getCluster().getMetadataQuery();
+ return HiveRelOptUtil.isRewritablePKFKJoin(join, join.getLeft(),
join.getRight(), mq).rewritable;
+ };
+ return operandJ(Join.class, null, isRewritable, any());
+ }
+
public static final HiveFilterProjectTransposeRule
DETERMINISTIC_WINDOWING_ON_NON_FILTERING_JOIN =
new HiveFilterProjectTransposeRule(
- operand(Filter.class, operand(Project.class, operand(Join.class,
any()))),
+ operand(Filter.class, operand(Project.class, joinOperand())),
HiveRelFactories.HIVE_BUILDER, true, true,
ProjectMergeRule.DEFAULT_BLOAT);
public static final HiveFilterProjectTransposeRule DETERMINISTIC_WINDOWING =
@@ -75,7 +86,7 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
public static final HiveFilterProjectTransposeRule
DETERMINISTIC_ON_NON_FILTERING_JOIN =
new HiveFilterProjectTransposeRule(
- operand(Filter.class, operand(Project.class, operand(Join.class,
any()))),
+ operand(Filter.class, operand(Project.class, joinOperand())),
HiveRelFactories.HIVE_BUILDER, true, false,
ProjectMergeRule.DEFAULT_BLOAT);
public static final HiveFilterProjectTransposeRule DETERMINISTIC =
@@ -86,6 +97,20 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
HiveRelFactories.HIVE_BUILDER, false, false,
ProjectMergeRule.DEFAULT_BLOAT);
+ public static final HiveFilterProjectTransposeRule SCAN = new
HiveFilterProjectTransposeRule(
+ operand(Filter.class, operand(HiveProject.class,
operand(HiveTableScan.class, none()))),
+ HiveRelFactories.HIVE_BUILDER,
+ false,
+ false,
+ ProjectMergeRule.DEFAULT_BLOAT);
+
+ public static final HiveFilterProjectTransposeRule DRUID = new
HiveFilterProjectTransposeRule(
+ operand(Filter.class, operand(HiveProject.class,
operand(DruidQuery.class, none()))),
+ HiveRelFactories.HIVE_BUILDER,
+ false,
+ false,
+ ProjectMergeRule.DEFAULT_BLOAT);
+
private final boolean onlyDeterministic;
private final boolean pushThroughWindowing;
@@ -127,16 +152,6 @@ public boolean matches(RelOptRuleCall call) {
if (this.onlyDeterministic && !HiveCalciteUtil.isDeterministic(condition))
{
return false;
}
-
- if (call.rels.length > 2) {
- final Join joinRel = call.rel(2);
- RewritablePKFKJoinInfo joinInfo = HiveRelOptUtil.isRewritablePKFKJoin(
- joinRel, joinRel.getLeft(), joinRel.getRight(),
call.getMetadataQuery());
- if (!joinInfo.rewritable) {
- return false;
- }
- }
-
return super.matches(call);
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java
index 3d507f4ebf9..f6f011fe582 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java
@@ -69,7 +69,6 @@
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin;
-import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import
org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange;
import
org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
@@ -390,10 +389,7 @@ private static RelNode project(DruidQuery dq,
ImmutableBitSet fieldsUsed,
true));
nameList.add(extraField.getName());
}
-
- HiveProject hp = (HiveProject) relBuilder.push(dq).project(exprList,
nameList).build();
- hp.setSynthetic();
- return hp;
+ return relBuilder.push(dq).project(exprList, nameList).build();
}
private boolean isRexLiteral(final RexNode rexNode) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index eb24496743f..f1e9da25ebe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -229,7 +229,6 @@
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFieldTrimmerRule;
import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterAggregateTransposeRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterJoinRule;
-import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTSTransposeRule;
import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTransposeRule;
import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterSetOpTransposeRule;
import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterSortPredicates;
@@ -1933,7 +1932,7 @@ protected RelNode applyPreJoinOrderingTransforms(RelNode
basePlan, RelMetadataPr
// matches FIL-PROJ-TS
// Also merge, remove and reduce Project if possible
generatePartialProgram(program, true, HepMatchOrder.TOP_DOWN,
- HiveFilterProjectTSTransposeRule.INSTANCE,
HiveFilterProjectTSTransposeRule.INSTANCE_DRUID,
+ HiveFilterProjectTransposeRule.SCAN,
HiveFilterProjectTransposeRule.DRUID,
HiveProjectFilterPullUpConstantsRule.INSTANCE,
HiveProjectMergeRule.INSTANCE,
ProjectRemoveRule.Config.DEFAULT.toRule(),
HiveSortMergeRule.INSTANCE);
@@ -2100,7 +2099,7 @@ private RelNode rewriteUsingViews(RelOptPlanner planner,
RelNode basePlan,
// Unboxing rule
planner.addRule(HiveMaterializedViewBoxing.INSTANCE_UNBOXING);
// Partition pruner rule
- planner.addRule(HiveFilterProjectTSTransposeRule.INSTANCE);
+ planner.addRule(HiveFilterProjectTransposeRule.SCAN);
planner.addRule(new HivePartitionPruneRule(conf));
// Optimize plan
@@ -2411,7 +2410,7 @@ private RelNode applyPostJoinOrderingTransform(RelNode
basePlan, RelMetadataProv
generatePartialProgram(program, false, HepMatchOrder.DEPTH_FIRST,
ProjectRemoveRule.Config.DEFAULT.toRule(), new
ProjectMergeRule(false, HiveRelFactories.HIVE_BUILDER));
generatePartialProgram(program, true, HepMatchOrder.TOP_DOWN,
- HiveFilterProjectTSTransposeRule.INSTANCE,
HiveFilterProjectTSTransposeRule.INSTANCE_DRUID,
+ HiveFilterProjectTransposeRule.SCAN,
HiveFilterProjectTransposeRule.DRUID,
HiveProjectFilterPullUpConstantsRule.INSTANCE);
// 9.2. Introduce exchange operators below join/multijoin operators
