This is an automated email from the ASF dual-hosted git repository.
zhenchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new f398d74cbd [CALCITE-7235] Support Flexible HEP and Volcano Planner
Rule Configuration in Quidem Tests
f398d74cbd is described below
commit f398d74cbd808737504e6bd844960b764baed1ad
Author: Zhen Chen <[email protected]>
AuthorDate: Thu Oct 23 16:10:39 2025 +0800
[CALCITE-7235] Support Flexible HEP and Volcano Planner Rule Configuration
in Quidem Tests
---
core/src/test/resources/sql/hep.iq | 117 +++++++++++++++++++++
.../java/org/apache/calcite/test/QuidemTest.java | 84 ++++++++++++++-
2 files changed, 199 insertions(+), 2 deletions(-)
diff --git a/core/src/test/resources/sql/hep.iq
b/core/src/test/resources/sql/hep.iq
new file mode 100644
index 0000000000..8c5b36fe90
--- /dev/null
+++ b/core/src/test/resources/sql/hep.iq
@@ -0,0 +1,117 @@
+# hep.iq - hep tests can customizable optimization rules for hep planner
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This documents the required format for .iq files containing hep planner rule
configurations
+#
+# SYNTAX:
+# -------
+# !set hep-rules "
+# +CoreRules.RULE_NAME1,
+# -CoreRules.RULE_NAME2,
+# +EnumerableRules.ENUMERABLE_RULE_NAME"
+#
+# RULES FORMAT SPECIFICATION:
+# ---------------------------
+# 1. Must begin with: !set hep-rules " (double quote)
+# 2. Each rule must be on its own line. All lines except last must end with
comma.
+# The last rule line must close with " (double quote) on final line
+# 3. Rules must start with - (remove) or + (add)
+#
+# RULE PROCESSING LOGIC:
+# ----------------------
+# 1. Rule prefix interpretation:
+# - CoreRules.RULE_NAME -> added/removed from hep program
+# - EnumerableRules.RULE_NAME -> added/removed from volcano program
+# - RULE_NAME (no prefix) -> treated as CoreRules, added/removed from hep
program
+# 2. Duplicate operations on the same rule: last operation wins
+# Example: "+Rule1,+Rule1,-Rule1" results in rule being removed
+# 3. Optimizer execution pipeline:
+# The optimizer uses a multi-program architecture with the following
execution order:
+# - HEP program — executed by HepPlanner; initial rules set by CoreRules.*
+# - Volcano program — executed by VolcanoPlanner; initial rules set by
EnumerableRules.ENUMERABLE_RULES
+# and modified by configurable EnumerableRules.* rule sets.
+# - Calc program — executed by HepPlanner; initial rules set by
RelOptRules.CALC_RULES (non-configurable)
+
+!use scott
+!set outputformat mysql
+
+# The hep-rules test is functioning correctly.
+!set hep-rules "
++CoreRules.PROJECT_FILTER_TRANSPOSE,
++CoreRules.UNION_FILTER_TO_FILTER"
+
+SELECT mgr, comm FROM emp WHERE mgr = 12
+UNION
+SELECT mgr, comm FROM emp WHERE comm > 5;
++------+---------+
+| MGR | COMM |
++------+---------+
+| 7698 | 1400.00 |
+| 7698 | 300.00 |
+| 7698 | 500.00 |
++------+---------+
+(3 rows)
+
+!ok
+EnumerableAggregate(group=[{0, 1}])
+ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER],
expr#9=[12], expr#10=[=($t8, $t9)], expr#11=[CAST($t6):DECIMAL(12, 2)],
expr#12=[5.00:DECIMAL(12, 2)], expr#13=[>($t11, $t12)], expr#14=[OR($t10,
$t13)], MGR=[$t3], COMM=[$t6], $condition=[$t14])
+ EnumerableTableScan(table=[[scott, EMP]])
+!plan
+!set hep-rules original
+
+# Testing with the planner-rules shows that due to cost-based selection issues,
+# the planner fails to choose a plan that includes the Aggregate operator.
+!set planner-rules "
++CoreRules.PROJECT_FILTER_TRANSPOSE,
++CoreRules.UNION_FILTER_TO_FILTER"
+
+SELECT mgr, comm FROM emp WHERE mgr = 12
+UNION
+SELECT mgr, comm FROM emp WHERE comm > 5;
++------+---------+
+| MGR | COMM |
++------+---------+
+| 7698 | 1400.00 |
+| 7698 | 300.00 |
+| 7698 | 500.00 |
++------+---------+
+(3 rows)
+
+!ok
+EnumerableUnion(all=[false])
+ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER],
expr#9=[12], expr#10=[=($t8, $t9)], MGR=[$t3], COMM=[$t6], $condition=[$t10])
+ EnumerableTableScan(table=[[scott, EMP]])
+ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)],
expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], MGR=[$t3], COMM=[$t6],
$condition=[$t10])
+ EnumerableTableScan(table=[[scott, EMP]])
+!plan
+!set planner-rules original
+
+# This hep-rules test validates support for modifying EnumerableRules.
+!set hep-rules "
++CoreRules.PROJECT_FILTER_TRANSPOSE,
++CoreRules.UNION_FILTER_TO_FILTER,
+-EnumerableRules.ENUMERABLE_AGGREGATE_RULE"
+
+SELECT mgr, comm FROM emp WHERE mgr = 12
+UNION
+SELECT mgr, comm FROM emp WHERE comm > 5;
+Missing conversion is LogicalAggregate[convention: NONE -> ENUMERABLE]
+!error
+!set hep-rules original
+
+# End hep.iq
diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
index 83419d6fb7..f55b2c060f 100644
--- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
@@ -24,6 +24,7 @@
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
import org.apache.calcite.rel.rules.CoreRules;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -41,8 +42,12 @@
import org.apache.calcite.test.schemata.catchall.CatchallSchema;
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.Planner;
+import org.apache.calcite.tools.Program;
+import org.apache.calcite.tools.Programs;
+import org.apache.calcite.tools.RuleSets;
import org.apache.calcite.util.Bug;
import org.apache.calcite.util.Closer;
+import org.apache.calcite.util.Holder;
import org.apache.calcite.util.Sources;
import org.apache.calcite.util.Util;
@@ -261,6 +266,30 @@ protected void checkRun(String path) throws Exception {
}));
}
}
+
+ if (propertyName.equals("hep-rules")) {
+ closer.add(
+ Hook.PROGRAM.addThread((Consumer<Holder<Program>>)
+ holder -> {
+ List<RelOptRule> hepRules = new ArrayList<>();
+ List<RelOptRule> volcanoRules =
+ new ArrayList<>(EnumerableRules.ENUMERABLE_RULES);
+
+ applyRulesInOrder((String) value, hepRules,
volcanoRules);
+
+ Program hepProgram =
+ Programs.hep(RuleSets.ofList(hepRules), false,
+ DefaultRelMetadataProvider.INSTANCE);
+ Program calcProgram =
+ Programs.calc(DefaultRelMetadataProvider.INSTANCE);
+ Program volcanoProgram =
+ Programs.of(RuleSets.ofList(volcanoRules));
+
+ Program combinedProgram =
+ Programs.sequence(hepProgram, volcanoProgram,
calcProgram);
+ holder.set(combinedProgram);
+ }));
+ }
})
.withEnv(QuidemTest::getEnv)
.build();
@@ -317,6 +346,49 @@ private static void parseRules(String value,
List<RelOptRule> rulesAdd,
}
}
+ /**
+ * Parse the rules string and apply operations in textual order to the
+ * provided hepRules and volcanoRules lists.
+ */
+ private static void applyRulesInOrder(String value,
+ List<RelOptRule> hepRules, List<RelOptRule> volcanoRules) {
+ Pattern pattern =
Pattern.compile("([+-])((CoreRules|EnumerableRules)\\.)?(\\w+)");
+ Matcher matcher = pattern.matcher(value);
+
+ while (matcher.find()) {
+ char operation = matcher.group(1).charAt(0);
+ String ruleSource = matcher.group(3);
+ String ruleName = matcher.group(4);
+
+ try {
+ RelOptRule rule;
+ boolean targetVolcano = false;
+ if (ruleSource == null || ruleSource.equals("CoreRules")) {
+ rule = getCoreRule(ruleName);
+ } else if (ruleSource.equals("EnumerableRules")) {
+ Object ruleObj = EnumerableRules.class.getField(ruleName).get(null);
+ rule = (RelOptRule) ruleObj;
+ targetVolcano = true;
+ } else {
+ throw new RuntimeException("Unknown rule: " + ruleName);
+ }
+
+ List<RelOptRule> target = targetVolcano ? volcanoRules : hepRules;
+ if (operation == '+') {
+ if (!target.contains(rule)) {
+ target.add(rule);
+ }
+ } else if (operation == '-') {
+ target.remove(rule);
+ } else {
+ throw new RuntimeException("unknown operation '" + operation + "'");
+ }
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException("set rules failed: " + e.getMessage(), e);
+ }
+ }
+ }
+
public static RelOptRule getCoreRule(String ruleName) {
try {
Field ruleField = CoreRules.class.getField(ruleName);
@@ -334,9 +406,17 @@ public static RelOptRule getCoreRule(String ruleName) {
private static void setRules(char operation, RelOptRule rule,
List<RelOptRule> rulesAdd, List<RelOptRule> rulesRemove) {
if (operation == '+') {
- rulesAdd.add(rule);
+ // Remove from remove list if present, then add to add list
+ rulesRemove.remove(rule);
+ if (!rulesAdd.contains(rule)) {
+ rulesAdd.add(rule);
+ }
} else if (operation == '-') {
- rulesRemove.add(rule);
+ // Remove from add list if present, then add to remove list
+ rulesAdd.remove(rule);
+ if (!rulesRemove.contains(rule)) {
+ rulesRemove.add(rule);
+ }
} else {
throw new RuntimeException("unknown operation '" + operation + "'");
}
