This is an automated email from the ASF dual-hosted git repository.
hyuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new f17367e [CALCITE-2970] Add AbstractConverter only between derived and
required traitset
f17367e is described below
commit f17367e6ae4814c035e234568e212ba23080e9dd
Author: Haisheng Yuan <[email protected]>
AuthorDate: Wed Mar 18 00:38:34 2020 -0500
[CALCITE-2970] Add AbstractConverter only between derived and required
traitset
Before this patch, the VolcanoPlanner couldn't distinguish traitset derived
from child operators and traitset required by parent operators.
AbstractConverters are added between all of these traitsets no matter it is
derived or required, which causes the explosion of search space. e.g.
SELECT a,b,c,max(d) FROM foo GROUP BY a,b,c;
Aggregate
+-- TableScan
For distributed system, suppose the Aggregate operator may require the
following traitsets from TableScan with exact match:
- Singleton distribution
- Hash distribution on a
- Hash distribution on b
- Hash distribution on c
- Hash distribution on a,b
- Hash distribution on b,c
- Hash distribution on a,c
- Hash distribution on a,b,c
VolcanoPlanner would add 7*7+8 = 57 abstract converters into the RelSet,
e.g.
abstractConverter between [a] and [b,c], even if the satisfying match is
allowed, e.g. distribution on [a] statisfy distribution on [a,b,c], there
are
still lots of abstract converters. But we only need 8.
This patch fixes above issue by adding state to RelSubset indicating whether
the added traitset is required or derived. The traitset can be both required
and derived. Only abstract converter from derived traitset to required
traitset
is added.
By default, when adding a new RelNode to RelSet, we treat its traitset as
derived, when calling changeTraits, the traitset will be treated as
required.
Unfortunately, almost all the RelNodes except AbstractConverter are added
through rule transformation, when the AbstractConverter is transformed to a
enforcing operator, e.g. PhysicalSort, the planner will still treat its
traitset as derived, which will trigger the creation of AbstractConverter
between this RelSubset and remaining RelSubsets in the RelSet. To avoid this
issue, though not clean but work, enforcing operator and AbstactConverter
should override isEnforcer() method to indicate the RelNode is added due to
the desired traitset is not satisfied. The user needs to judge by his/her
own
whether to mark enforcing operator.
Close #1860
---
.../adapter/enumerable/EnumerableConvention.java | 2 +-
.../java/org/apache/calcite/plan/Convention.java | 10 +-
.../calcite/plan/volcano/AbstractConverter.java | 4 +
.../org/apache/calcite/plan/volcano/RelSet.java | 152 +++++++++++----------
.../org/apache/calcite/plan/volcano/RelSubset.java | 28 +++-
.../calcite/plan/volcano/VolcanoPlanner.java | 3 +-
.../org/apache/calcite/rel/AbstractRelNode.java | 4 +
.../main/java/org/apache/calcite/rel/RelNode.java | 11 ++
.../java/org/apache/calcite/rel/core/Sort.java | 5 +
.../calcite/rel/metadata/RelMdCollation.java | 6 +
.../java/org/apache/calcite/test/JdbcTest.java | 26 ++--
.../java/org/apache/calcite/test/StreamTest.java | 18 +--
.../test/enumerable/EnumerableCorrelateTest.java | 2 +
.../test/enumerable/EnumerableHashJoinTest.java | 7 +
core/src/test/resources/sql/misc.iq | 41 +++---
.../org/apache/calcite/adapter/file/SqlTest.java | 3 +-
.../org/apache/calcite/test/PigAdapterTest.java | 5 +
17 files changed, 211 insertions(+), 116 deletions(-)
diff --git
a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConvention.java
b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConvention.java
index 41bb8e5..c07bb0b 100644
---
a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConvention.java
+++
b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConvention.java
@@ -62,6 +62,6 @@ public enum EnumerableConvention implements Convention {
public boolean useAbstractConvertersForConversion(RelTraitSet fromTraits,
RelTraitSet toTraits) {
- return false;
+ return true;
}
}
diff --git a/core/src/main/java/org/apache/calcite/plan/Convention.java
b/core/src/main/java/org/apache/calcite/plan/Convention.java
index 5ad1847..e4df920 100644
--- a/core/src/main/java/org/apache/calcite/plan/Convention.java
+++ b/core/src/main/java/org/apache/calcite/plan/Convention.java
@@ -44,7 +44,9 @@ public interface Convention extends RelTrait {
* @param toConvention Desired convention to convert to
* @return Whether we should convert from this convention to toConvention
*/
- boolean canConvertConvention(Convention toConvention);
+ default boolean canConvertConvention(Convention toConvention) {
+ return false;
+ }
/**
* Returns whether we should convert from this trait set to the other trait
@@ -59,8 +61,10 @@ public interface Convention extends RelTrait {
* @param toTraits Target traits
* @return Whether we should add converters
*/
- boolean useAbstractConvertersForConversion(RelTraitSet fromTraits,
- RelTraitSet toTraits);
+ default boolean useAbstractConvertersForConversion(RelTraitSet fromTraits,
+ RelTraitSet toTraits) {
+ return true;
+ }
/**
* Default implementation.
diff --git
a/core/src/main/java/org/apache/calcite/plan/volcano/AbstractConverter.java
b/core/src/main/java/org/apache/calcite/plan/volcano/AbstractConverter.java
index 0afdd69..f1ef0b9 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/AbstractConverter.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/AbstractConverter.java
@@ -81,6 +81,10 @@ public class AbstractConverter extends ConverterImpl {
return pw;
}
+ @Override public boolean isEnforcer() {
+ return true;
+ }
+
//~ Inner Classes ----------------------------------------------------------
/**
diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java
b/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java
index 0314345..9477094 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java
@@ -16,6 +16,7 @@
*/
package org.apache.calcite.plan.volcano;
+import org.apache.calcite.plan.Convention;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptListener;
import org.apache.calcite.plan.RelOptUtil;
@@ -38,6 +39,7 @@ import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* A <code>RelSet</code> is an equivalence-set of expressions; that is, a set
of
@@ -160,107 +162,106 @@ class RelSet {
public RelSubset add(RelNode rel) {
assert equivalentSet == null : "adding to a dead set";
final RelTraitSet traitSet = rel.getTraitSet().simplify();
- final RelSubset subset = getOrCreateSubset(rel.getCluster(), traitSet);
+ final RelSubset subset = getOrCreateSubset(
+ rel.getCluster(), traitSet, rel.isEnforcer());
subset.add(rel);
return subset;
}
+ /**
+ * If the subset is required, convert derived subsets to this subset.
+ * Otherwise, convert this subset to required subsets in this RelSet.
+ * The subset can be both required and derived.
+ */
private void addAbstractConverters(
- VolcanoPlanner planner, RelOptCluster cluster, RelSubset subset, boolean
subsetToOthers) {
- // Converters from newly introduced subset to all the remaining one (vice
versa), only if
- // we can convert. No point adding converters if it is not possible.
- for (RelSubset other : subsets) {
+ RelOptCluster cluster, RelSubset subset, boolean required) {
+ List<RelSubset> others = subsets.stream().filter(
+ n -> required ? n.isDerived() : n.isRequired())
+ .collect(Collectors.toList());
+ for (RelSubset other : others) {
assert other.getTraitSet().size() == subset.getTraitSet().size();
+ RelSubset from = subset;
+ RelSubset to = other;
+
+ if (required) {
+ from = other;
+ to = subset;
+ }
- if ((other == subset)
- || (subsetToOthers
- && !subset.getConvention().useAbstractConvertersForConversion(
- subset.getTraitSet(), other.getTraitSet()))
- || (!subsetToOthers
- && !other.getConvention().useAbstractConvertersForConversion(
- other.getTraitSet(), subset.getTraitSet()))) {
+ if (from == to || !from.getConvention()
+ .useAbstractConvertersForConversion(
+ from.getTraitSet(), to.getTraitSet())) {
continue;
}
final ImmutableList<RelTrait> difference =
- subset.getTraitSet().difference(other.getTraitSet());
+ to.getTraitSet().difference(from.getTraitSet());
- boolean addAbstractConverter = true;
- int numTraitNeedConvert = 0;
-
- for (RelTrait curOtherTrait : difference) {
- RelTraitDef traitDef = curOtherTrait.getTraitDef();
- RelTrait curRelTrait = subset.getTraitSet().getTrait(traitDef);
-
- if (curRelTrait == null) {
- addAbstractConverter = false;
- break;
- }
+ boolean needsConverter = false;
- assert curRelTrait.getTraitDef() == traitDef;
-
- boolean canConvert = false;
- boolean needConvert = false;
- if (subsetToOthers) {
- // We can convert from subset to other. So, add converter with
subset as child and
- // traitset as the other's traitset.
- canConvert = traitDef.canConvert(
- cluster.getPlanner(), curRelTrait, curOtherTrait, subset);
- needConvert = !curRelTrait.satisfies(curOtherTrait);
- } else {
- // We can convert from others to subset.
- canConvert = traitDef.canConvert(
- cluster.getPlanner(), curOtherTrait, curRelTrait, other);
- needConvert = !curOtherTrait.satisfies(curRelTrait);
- }
+ for (RelTrait fromTrait : difference) {
+ RelTraitDef traitDef = fromTrait.getTraitDef();
+ RelTrait toTrait = to.getTraitSet().getTrait(traitDef);
- if (!canConvert) {
- addAbstractConverter = false;
+ if (toTrait == null || !traitDef.canConvert(
+ cluster.getPlanner(), fromTrait, toTrait, from)) {
+ needsConverter = false;
break;
}
- if (needConvert) {
- numTraitNeedConvert++;
+ if (!fromTrait.satisfies(toTrait)) {
+ needsConverter = true;
}
}
- if (addAbstractConverter && numTraitNeedConvert > 0) {
- if (subsetToOthers) {
- final AbstractConverter converter =
- new AbstractConverter(cluster, subset, null,
other.getTraitSet());
- planner.register(converter, other);
- } else {
- final AbstractConverter converter =
- new AbstractConverter(cluster, other, null,
subset.getTraitSet());
- planner.register(converter, subset);
- }
+ if (needsConverter) {
+ final AbstractConverter converter =
+ new AbstractConverter(cluster, from, null, to.getTraitSet());
+ cluster.getPlanner().register(converter, to);
}
}
}
+ RelSubset getOrCreateSubset(RelOptCluster cluster, RelTraitSet traits) {
+ return getOrCreateSubset(cluster, traits, false);
+ }
+
RelSubset getOrCreateSubset(
- RelOptCluster cluster,
- RelTraitSet traits) {
+ RelOptCluster cluster, RelTraitSet traits, boolean required) {
+ boolean needsConverter = false;
RelSubset subset = getSubset(traits);
+
if (subset == null) {
+ needsConverter = true;
subset = new RelSubset(cluster, this, traits);
- final VolcanoPlanner planner =
- (VolcanoPlanner) cluster.getPlanner();
-
- addAbstractConverters(planner, cluster, subset, true);
-
- // Need to first add to subset before adding the abstract converters
(for others->subset)
- // since otherwise during register() the planner will try to add this
subset again.
+ // Need to first add to subset before adding the abstract
+ // converters (for others->subset), since otherwise during
+ // register() the planner will try to add this subset again.
subsets.add(subset);
- addAbstractConverters(planner, cluster, subset, false);
-
+ final VolcanoPlanner planner = (VolcanoPlanner) cluster.getPlanner();
if (planner.listener != null) {
postEquivalenceEvent(planner, subset);
}
+ } else if ((required && !subset.isRequired())
+ || (!required && !subset.isDerived())) {
+ needsConverter = true;
+ }
+
+ if (subset.getConvention() == Convention.NONE) {
+ needsConverter = false;
+ } else if (required) {
+ subset.setRequired();
+ } else {
+ subset.setDerived();
+ }
+
+ if (needsConverter) {
+ addAbstractConverters(cluster, subset, required);
}
+
return subset;
}
@@ -327,7 +328,8 @@ class RelSet {
assert otherSet.equivalentSet == null;
LOGGER.trace("Merge set#{} into set#{}", otherSet.id, id);
otherSet.equivalentSet = this;
- RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
+ RelOptCluster cluster = rel.getCluster();
+ RelMetadataQuery mq = cluster.getMetadataQuery();
// remove from table
boolean existed = planner.allSets.remove(otherSet);
@@ -337,10 +339,20 @@ class RelSet {
// merge subsets
for (RelSubset otherSubset : otherSet.subsets) {
- RelSubset subset =
- getOrCreateSubset(
- otherSubset.getCluster(),
- otherSubset.getTraitSet());
+ RelSubset subset = null;
+ RelTraitSet otherTraits = otherSubset.getTraitSet();
+
+ // If it is logical or derived physical traitSet
+ if (otherSubset.isDerived() || !otherSubset.isRequired()) {
+ subset = getOrCreateSubset(cluster, otherTraits, false);
+ }
+
+ // It may be required only, or both derived and required,
+ // in which case, register again.
+ if (otherSubset.isRequired()) {
+ subset = getOrCreateSubset(cluster, otherTraits, true);
+ }
+
// collect RelSubset instances, whose best should be changed
if (otherSubset.bestCost.isLt(subset.bestCost)) {
changedSubsets.put(subset, otherSubset.best);
diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java
b/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java
index 80bbab8..0375955 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java
@@ -74,6 +74,8 @@ public class RelSubset extends AbstractRelNode {
//~ Static fields/initializers ---------------------------------------------
private static final Logger LOGGER = CalciteTrace.getPlannerTracer();
+ private static final int DERIVED = 1;
+ private static final int REQUIRED = 2;
//~ Instance fields --------------------------------------------------------
@@ -98,10 +100,13 @@ public class RelSubset extends AbstractRelNode {
long timestamp;
/**
- * Flag indicating whether this RelSubset's importance was artificially
- * boosted.
+ * Physical property state of current subset
+ * 0: logical operators, NONE convention is neither DERIVED nor REQUIRED
+ * 1: traitSet DERIVED from child operators or itself
+ * 2: traitSet REQUIRED from parent operators
+ * 3: both DERIVED and REQUIRED
*/
- boolean boosted;
+ private int state = 0;
//~ Constructors -----------------------------------------------------------
@@ -111,7 +116,6 @@ public class RelSubset extends AbstractRelNode {
RelTraitSet traits) {
super(cluster, traits);
this.set = set;
- this.boosted = false;
assert traits.allSimple();
computeBestCost(cluster.getPlanner());
recomputeDigest();
@@ -144,6 +148,22 @@ public class RelSubset extends AbstractRelNode {
}
}
+ void setDerived() {
+ state |= DERIVED;
+ }
+
+ void setRequired() {
+ state |= REQUIRED;
+ }
+
+ public boolean isDerived() {
+ return (state & DERIVED) == DERIVED;
+ }
+
+ public boolean isRequired() {
+ return (state & REQUIRED) == REQUIRED;
+ }
+
public RelNode getBest() {
return best;
}
diff --git
a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
index ab21319..ef631d5 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
@@ -495,7 +495,8 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
return rel2;
}
- return rel2.set.getOrCreateSubset(rel.getCluster(), toTraits.simplify());
+ return rel2.set.getOrCreateSubset(
+ rel.getCluster(), toTraits, true);
}
public RelOptPlanner chooseDelegate() {
diff --git a/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java
b/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java
index 703a782..8240c60 100644
--- a/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java
+++ b/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java
@@ -248,6 +248,10 @@ public abstract class AbstractRelNode implements RelNode {
// for default case, nothing to do
}
+ public boolean isEnforcer() {
+ return false;
+ }
+
public void collectVariablesSet(Set<CorrelationId> variableSet) {
}
diff --git a/core/src/main/java/org/apache/calcite/rel/RelNode.java
b/core/src/main/java/org/apache/calcite/rel/RelNode.java
index 362e885..11c35ef 100644
--- a/core/src/main/java/org/apache/calcite/rel/RelNode.java
+++ b/core/src/main/java/org/apache/calcite/rel/RelNode.java
@@ -408,6 +408,17 @@ public interface RelNode extends RelOptNode, Cloneable {
void register(RelOptPlanner planner);
/**
+ * Indicates whether it is an enforcer operator, e.g. PhysicalSort,
+ * PhysicalHashDistribute, etc. As an enforcer, the operator must be
+ * created only when required traitSet is not satisfied by its input.
+ *
+ * @return Whether it is an enforcer operator
+ */
+ default boolean isEnforcer() {
+ return false;
+ }
+
+ /**
* Returns whether the result of this relational expression is uniquely
* identified by this columns with the given ordinals.
*
diff --git a/core/src/main/java/org/apache/calcite/rel/core/Sort.java
b/core/src/main/java/org/apache/calcite/rel/core/Sort.java
index 8611240..4bbcdd8 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/Sort.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/Sort.java
@@ -157,6 +157,11 @@ public abstract class Sort extends SingleRel {
return copy(traitSet, getInput(), collation, offset, fetch);
}
+ @Override public boolean isEnforcer() {
+ return offset == null && fetch == null
+ && collation.getFieldCollations().size() > 0;
+ }
+
/**
* Returns the array of {@link RelFieldCollation}s asked for by the sort
* specification, from most significant to least significant.
diff --git
a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java
b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java
index b83df05..5b7b8ba 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java
@@ -20,6 +20,7 @@ import
org.apache.calcite.adapter.enumerable.EnumerableCorrelate;
import org.apache.calcite.adapter.enumerable.EnumerableHashJoin;
import org.apache.calcite.adapter.enumerable.EnumerableMergeJoin;
import org.apache.calcite.adapter.enumerable.EnumerableNestedLoopJoin;
+import org.apache.calcite.adapter.jdbc.JdbcToEnumerableConverter;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.hep.HepRelVertex;
@@ -203,6 +204,11 @@ public class RelMdCollation
values(mq, values.getRowType(), values.getTuples()));
}
+ public ImmutableList<RelCollation> collations(JdbcToEnumerableConverter rel,
+ RelMetadataQuery mq) {
+ return mq.collations(rel.getInput());
+ }
+
public ImmutableList<RelCollation> collations(HepRelVertex rel,
RelMetadataQuery mq) {
return mq.collations(rel.getCurrentRel());
diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java
b/core/src/test/java/org/apache/calcite/test/JdbcTest.java
index ca923d1..c234e9e 100644
--- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java
+++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java
@@ -1129,11 +1129,13 @@ public class JdbcTest {
+ "and p.\"brand_name\" = 'Washington'")
.explainMatches("including all attributes ",
CalciteAssert.checkMaskedResultContains(""
- + "EnumerableHashJoin(condition=[=($0, $38)],
joinType=[inner]): rowcount = 7.050660528307499E8, cumulative cost =
{1.0640240216183146E9 rows, 777302.0 cpu, 0.0 io}\n"
- + " EnumerableHashJoin(condition=[=($2, $8)],
joinType=[inner]): rowcount = 2.0087351932499997E7, cumulative cost =
{2.117504719375143E7 rows, 724261.0 cpu, 0.0 io}\n"
- + " EnumerableTableScan(table=[[foodmart2,
sales_fact_1997]]): rowcount = 86837.0, cumulative cost = {86837.0 rows,
86838.0 cpu, 0.0 io}\n"
- + " EnumerableCalc(expr#0..28=[{inputs}], expr#29=['San
Francisco':VARCHAR(30)], expr#30=[=($t9, $t29)], proj#0..28=[{exprs}],
$condition=[$t30]): rowcount = 1542.1499999999999, cumulative cost = {11823.15
rows, 637423.0 cpu, 0.0 io}\n"
- + " EnumerableTableScan(table=[[foodmart2, customer]]):
rowcount = 10281.0, cumulative cost = {10281.0 rows, 10282.0 cpu, 0.0 io}\n"
+ + "EnumerableMergeJoin(condition=[=($0, $38)],
joinType=[inner]): rowcount = 7.050660528307499E8, cumulative cost =
{7.656040129282498E8 rows, 5.0023949296644424E10 cpu, 0.0 io}\n"
+ + " EnumerableSort(sort0=[$0], dir0=[ASC]): rowcount =
2.0087351932499997E7, cumulative cost = {4.044858016499999E7 rows,
5.0023896255644424E10 cpu, 0.0 io}\n"
+ + " EnumerableMergeJoin(condition=[=($2, $8)],
joinType=[inner]): rowcount = 2.0087351932499997E7, cumulative cost =
{2.0361228232499994E7 rows, 3.232400376004586E7 cpu, 0.0 io}\n"
+ + " EnumerableSort(sort0=[$2], dir0=[ASC]): rowcount =
86837.0, cumulative cost = {173674.0 rows, 3.168658076004586E7 cpu, 0.0 io}\n"
+ + " EnumerableTableScan(table=[[foodmart2,
sales_fact_1997]]): rowcount = 86837.0, cumulative cost = {86837.0 rows,
86838.0 cpu, 0.0 io}\n"
+ + " EnumerableCalc(expr#0..28=[{inputs}], expr#29=['San
Francisco':VARCHAR(30)], expr#30=[=($t9, $t29)], proj#0..28=[{exprs}],
$condition=[$t30]): rowcount = 1542.1499999999999, cumulative cost = {11823.15
rows, 637423.0 cpu, 0.0 io}\n"
+ + " EnumerableTableScan(table=[[foodmart2, customer]]):
rowcount = 10281.0, cumulative cost = {10281.0 rows, 10282.0 cpu, 0.0 io}\n"
+ " EnumerableCalc(expr#0..14=[{inputs}],
expr#15=['Washington':VARCHAR(60)], expr#16=[=($t2, $t15)],
proj#0..14=[{exprs}], $condition=[$t16]): rowcount = 234.0, cumulative cost =
{1794.0 rows, 53041.0 cpu, 0.0 io}\n"
+ " EnumerableTableScan(table=[[foodmart2, product]]):
rowcount = 1560.0, cumulative cost = {1560.0 rows, 1561.0 cpu, 0.0 io}\n"));
}
@@ -1836,7 +1838,7 @@ public class JdbcTest {
// 13 116 - OOM did not complete
checkJoinNWay(1);
checkJoinNWay(3);
- checkJoinNWay(6);
+ checkJoinNWay(13);
}
private static void checkJoinNWay(int n) {
@@ -2737,11 +2739,13 @@ public class JdbcTest {
+ " join \"hr\".\"depts\" using (\"deptno\")")
.explainContains(""
+ "EnumerableCalc(expr#0..3=[{inputs}], empid=[$t0], deptno=[$t2],
name=[$t3])\n"
- + " EnumerableHashJoin(condition=[=($1, $2)], joinType=[inner])\n"
- + " EnumerableCalc(expr#0..4=[{inputs}], proj#0..1=[{exprs}])\n"
- + " EnumerableTableScan(table=[[hr, emps]])\n"
- + " EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])\n"
- + " EnumerableTableScan(table=[[hr, depts]])")
+ + " EnumerableMergeJoin(condition=[=($1, $2)],
joinType=[inner])\n"
+ + " EnumerableSort(sort0=[$1], dir0=[ASC])\n"
+ + " EnumerableCalc(expr#0..4=[{inputs}],
proj#0..1=[{exprs}])\n"
+ + " EnumerableTableScan(table=[[hr, emps]])\n"
+ + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+ + " EnumerableCalc(expr#0..3=[{inputs}],
proj#0..1=[{exprs}])\n"
+ + " EnumerableTableScan(table=[[hr, depts]])")
.returns("empid=100; deptno=10; name=Sales\n"
+ "empid=150; deptno=10; name=Sales\n"
+ "empid=110; deptno=10; name=Sales\n");
diff --git a/core/src/test/java/org/apache/calcite/test/StreamTest.java
b/core/src/test/java/org/apache/calcite/test/StreamTest.java
index 076c218..11271d7 100644
--- a/core/src/test/java/org/apache/calcite/test/StreamTest.java
+++ b/core/src/test/java/org/apache/calcite/test/StreamTest.java
@@ -286,15 +286,17 @@ public class StreamTest {
+ " LogicalTableScan(table=[[STREAM_JOINS, PRODUCTS]])\n")
.explainContains(""
+ "EnumerableCalc(expr#0..6=[{inputs}], proj#0..1=[{exprs}],
SUPPLIERID=[$t6])\n"
- + " EnumerableHashJoin(condition=[=($4, $5)], joinType=[inner])\n"
- + " EnumerableCalc(expr#0..3=[{inputs}],
expr#4=[CAST($t2):VARCHAR(32) NOT NULL], proj#0..4=[{exprs}])\n"
- + " EnumerableInterpreter\n"
- + " BindableTableScan(table=[[STREAM_JOINS, ORDERS,
(STREAM)]])\n"
- + " EnumerableTableScan(table=[[STREAM_JOINS, PRODUCTS]])")
+ + " EnumerableMergeJoin(condition=[=($4, $5)],
joinType=[inner])\n"
+ + " EnumerableSort(sort0=[$4], dir0=[ASC])\n"
+ + " EnumerableCalc(expr#0..3=[{inputs}],
expr#4=[CAST($t2):VARCHAR(32) NOT NULL], proj#0..4=[{exprs}])\n"
+ + " EnumerableInterpreter\n"
+ + " BindableTableScan(table=[[STREAM_JOINS, ORDERS,
(STREAM)]])\n"
+ + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+ + " EnumerableTableScan(table=[[STREAM_JOINS, PRODUCTS]])\n")
.returns(
- startsWith("ROWTIME=2015-02-15 10:15:00; ORDERID=1; SUPPLIERID=1",
- "ROWTIME=2015-02-15 10:24:15; ORDERID=2; SUPPLIERID=0",
- "ROWTIME=2015-02-15 10:24:45; ORDERID=3; SUPPLIERID=1"));
+ startsWith("ROWTIME=2015-02-15 10:24:45; ORDERID=3; SUPPLIERID=1",
+ "ROWTIME=2015-02-15 10:15:00; ORDERID=1; SUPPLIERID=1",
+ "ROWTIME=2015-02-15 10:58:00; ORDERID=4; SUPPLIERID=1"));
}
@Disabled
diff --git
a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
index 4d80e09..11aa4aa 100644
---
a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
+++
b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
@@ -93,6 +93,7 @@ class EnumerableCorrelateTest {
// instead of EnumerableHashJoin(SEMI)
planner.addRule(JoinToCorrelateRule.INSTANCE);
planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE);
+ planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE);
})
.explainContains(""
+ "EnumerableCalc(expr#0..3=[{inputs}], empid=[$t1], name=[$t3])\n"
@@ -122,6 +123,7 @@ class EnumerableCorrelateTest {
planner.addRule(JoinToCorrelateRule.INSTANCE);
planner.addRule(FilterCorrelateRule.INSTANCE);
planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE);
+ planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE);
})
.explainContains(""
+ "EnumerableCalc(expr#0..3=[{inputs}], empid=[$t1], name=[$t3])\n"
diff --git
a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java
b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java
index afc86d9..8870f65 100644
---
a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java
+++
b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java
@@ -16,15 +16,20 @@
*/
package org.apache.calcite.test.enumerable;
+import org.apache.calcite.adapter.enumerable.EnumerableRules;
import org.apache.calcite.adapter.java.ReflectiveSchema;
import org.apache.calcite.config.CalciteConnectionProperty;
import org.apache.calcite.config.Lex;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.runtime.Hook;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.test.CalciteAssert;
import org.apache.calcite.test.JdbcTest;
import org.junit.jupiter.api.Test;
+import java.util.function.Consumer;
+
/**
* Unit test for
* {@link org.apache.calcite.adapter.enumerable.EnumerableHashJoin}.
@@ -36,6 +41,8 @@ class EnumerableHashJoinTest {
.query(
"select e.empid, e.name, d.name as dept from emps e join depts "
+ "d on e.deptno=d.deptno")
+ .withHook(Hook.PLANNER, (Consumer<RelOptPlanner>) planner ->
+ planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE))
.explainContains("EnumerableCalc(expr#0..4=[{inputs}], empid=[$t0], "
+ "name=[$t2], dept=[$t4])\n"
+ " EnumerableHashJoin(condition=[=($1, $3)], joinType=[inner])\n"
diff --git a/core/src/test/resources/sql/misc.iq
b/core/src/test/resources/sql/misc.iq
index a142fbf..dfd07e4 100644
--- a/core/src/test/resources/sql/misc.iq
+++ b/core/src/test/resources/sql/misc.iq
@@ -291,11 +291,13 @@ and e."name" <> d."name";
!ok
EnumerableCalc(expr#0..4=[{inputs}], empid=[$t0], name=[$t4], name0=[$t2])
- EnumerableHashJoin(condition=[AND(=($1, $3), <>(CAST($2):VARCHAR,
CAST($4):VARCHAR))], joinType=[inner])
- EnumerableCalc(expr#0..4=[{inputs}], proj#0..2=[{exprs}])
- EnumerableTableScan(table=[[hr, emps]])
- EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])
- EnumerableTableScan(table=[[hr, depts]])
+ EnumerableMergeJoin(condition=[AND(=($1, $3), <>(CAST($2):VARCHAR,
CAST($4):VARCHAR))], joinType=[inner])
+ EnumerableSort(sort0=[$1], dir0=[ASC])
+ EnumerableCalc(expr#0..4=[{inputs}], proj#0..2=[{exprs}])
+ EnumerableTableScan(table=[[hr, emps]])
+ EnumerableSort(sort0=[$0], dir0=[ASC])
+ EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])
+ EnumerableTableScan(table=[[hr, depts]])
!plan
# Same query, expressed using WHERE.
@@ -315,11 +317,13 @@ and e."name" <> d."name";
!ok
EnumerableCalc(expr#0..4=[{inputs}], empid=[$t0], name=[$t4], name0=[$t2])
- EnumerableHashJoin(condition=[AND(=($1, $3), <>(CAST($2):VARCHAR,
CAST($4):VARCHAR))], joinType=[inner])
- EnumerableCalc(expr#0..4=[{inputs}], proj#0..2=[{exprs}])
- EnumerableTableScan(table=[[hr, emps]])
- EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])
- EnumerableTableScan(table=[[hr, depts]])
+ EnumerableMergeJoin(condition=[AND(=($1, $3), <>(CAST($2):VARCHAR,
CAST($4):VARCHAR))], joinType=[inner])
+ EnumerableSort(sort0=[$1], dir0=[ASC])
+ EnumerableCalc(expr#0..4=[{inputs}], proj#0..2=[{exprs}])
+ EnumerableTableScan(table=[[hr, emps]])
+ EnumerableSort(sort0=[$0], dir0=[ASC])
+ EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])
+ EnumerableTableScan(table=[[hr, depts]])
!plan
# Un-correlated EXISTS
@@ -660,11 +664,13 @@ from "sales_fact_1997" as s
join "customer" as c on s."customer_id" = c."customer_id"
join "product" as p on s."product_id" = p."product_id"
where c."city" = 'San Francisco';
-EnumerableHashJoin(condition=[=($0, $38)], joinType=[inner])
- EnumerableHashJoin(condition=[=($2, $8)], joinType=[inner])
- EnumerableTableScan(table=[[foodmart2, sales_fact_1997]])
- EnumerableCalc(expr#0..28=[{inputs}], expr#29=['San
Francisco':VARCHAR(30)], expr#30=[=($t9, $t29)], proj#0..28=[{exprs}],
$condition=[$t30])
- EnumerableTableScan(table=[[foodmart2, customer]])
+EnumerableMergeJoin(condition=[=($0, $38)], joinType=[inner])
+ EnumerableSort(sort0=[$0], dir0=[ASC])
+ EnumerableMergeJoin(condition=[=($2, $8)], joinType=[inner])
+ EnumerableSort(sort0=[$2], dir0=[ASC])
+ EnumerableTableScan(table=[[foodmart2, sales_fact_1997]])
+ EnumerableCalc(expr#0..28=[{inputs}], expr#29=['San
Francisco':VARCHAR(30)], expr#30=[=($t9, $t29)], proj#0..28=[{exprs}],
$condition=[$t30])
+ EnumerableTableScan(table=[[foodmart2, customer]])
EnumerableTableScan(table=[[foodmart2, product]])
!plan
@@ -689,8 +695,9 @@ EnumerableCalc(expr#0..56=[{inputs}], product_id0=[$t20],
time_id=[$t21], custom
EnumerableCalc(expr#0..4=[{inputs}], expr#5=['Snacks':VARCHAR(30)],
expr#6=[=($t3, $t5)], proj#0..4=[{exprs}], $condition=[$t6])
EnumerableTableScan(table=[[foodmart2, product_class]])
EnumerableTableScan(table=[[foodmart2, product]])
- EnumerableHashJoin(condition=[=($2, $8)], joinType=[inner])
- EnumerableTableScan(table=[[foodmart2, sales_fact_1997]])
+ EnumerableMergeJoin(condition=[=($2, $8)], joinType=[inner])
+ EnumerableSort(sort0=[$2], dir0=[ASC])
+ EnumerableTableScan(table=[[foodmart2, sales_fact_1997]])
EnumerableCalc(expr#0..28=[{inputs}], expr#29=['San
Francisco':VARCHAR(30)], expr#30=[=($t9, $t29)], proj#0..28=[{exprs}],
$condition=[$t30])
EnumerableTableScan(table=[[foodmart2, customer]])
!plan
diff --git a/file/src/test/java/org/apache/calcite/adapter/file/SqlTest.java
b/file/src/test/java/org/apache/calcite/adapter/file/SqlTest.java
index 971878c..444f225 100644
--- a/file/src/test/java/org/apache/calcite/adapter/file/SqlTest.java
+++ b/file/src/test/java/org/apache/calcite/adapter/file/SqlTest.java
@@ -405,7 +405,8 @@ class SqlTest {
+ " NAME,\n"
+ " \"DATE\".JOINEDAT\n"
+ " from \"DATE\"\n"
- + "join emps on emps.empno = \"DATE\".EMPNO limit 3";
+ + "join emps on emps.empno = \"DATE\".EMPNO\n"
+ + "order by empno, name, joinedat limit 3";
final String[] lines = {
"EMPNO=100; NAME=Fred; JOINEDAT=1996-08-03",
"EMPNO=110; NAME=Eric; JOINEDAT=2001-01-01",
diff --git a/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java
b/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java
index 88ff742..aa8dc98 100644
--- a/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java
+++ b/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java
@@ -16,6 +16,9 @@
*/
package org.apache.calcite.test;
+import org.apache.calcite.adapter.enumerable.EnumerableRules;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.runtime.Hook;
import org.apache.calcite.util.Sources;
import com.google.common.collect.ImmutableMap;
@@ -145,6 +148,8 @@ class PigAdapterTest extends AbstractPigTest {
CalciteAssert.that()
.with(MODEL)
.query("select * from \"t\" join \"s\" on \"tc1\"=\"sc0\"")
+ .withHook(Hook.PLANNER, (Consumer<RelOptPlanner>) planner ->
+ planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE))
.explainContains("PigToEnumerableConverter\n"
+ " PigJoin(condition=[=($1, $2)], joinType=[inner])\n"
+ " PigTableScan(table=[[PIG, t]])\n"
