github-actions[bot] commented on code in PR #66482:
URL: https://github.com/apache/doris/pull/66482#discussion_r3773449843
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java:
##########
@@ -503,6 +540,199 @@ private Pair<LogicalPlan, Optional<Expression>>
addApply(SubqueryExpr subquery,
return Pair.of(logicalProject, newCorrelatedOuterExpr);
}
+ /**
+ * simplify the conjunct that contains mark join slots and infer the
behavior of each
+ * mark join slot, return the rewritten conjunct together with the mark
slots info.
+ *
+ * for each mark slot, the pair in the returned map has:
+ * Pair.first: whether the null and false values of the mark slot are
indistinguishable,
+ * i.e. the mark slot can be treated as a non-nullable
boolean. it only affects
+ * how the mark value is computed (treating null as false) and
never changes the
+ * number of output rows, so it's safe for every join type and
the filter.
+ * Pair.second: whether the original mark join can be directly eliminated
and turned into a
+ * plain semi join. a plain semi join only outputs the
matched rows, while a
+ * mark join keeps all original rows and adds a mark column,
so eliminating the
+ * mark join is only safe when discarding the unmatched rows
is already part of
+ * the containing join's semantics (inner, cross and semi
joins).
+ *
+ * when Pair.second is true, the mark slot is replaced by the true literal
in the returned
+ * conjunct, and the caller can drop the mark join slot to turn the mark
join into a plain
+ * semi join.
+ *
+ * extraEvaluationDomain extends the evaluation domain with expressions
that are not yet
+ * part of the plan but will be evaluated on the same rows later, e.g. the
generated
+ * assert_true(count(*) <= 1) that addApply synthesizes for a later
correlated scalar
+ * subquery; see collectGeneratedAssertionsOfLaterConjuncts.
+ *
+ * currentConjunctSubqueries are the subqueries of the conjunct being
processed: their own
+ * query plans are evaluated identically whether the mark join is kept or
eliminated (both
+ * the apply and the resulting semi/anti join evaluate the inner plan,
only the output row
+ * set differs), so a sensitive expression inside them cannot be affected
by the
+ * elimination and must be excluded from the evaluation domain; see
collectEvaluationDomain.
+ */
+ private Pair<Expression, Map<MarkJoinSlotReference, Pair<Boolean,
Boolean>>> simplifyConjunctWithMarkJoinSlot(
+ Expression conjunct, Plan plan, CascadesContext cascadesContext,
+ List<Expression> extraEvaluationDomain, Set<SubqueryExpr>
currentConjunctSubqueries) {
+ ExpressionRewriteContext rewriteContext = new
ExpressionRewriteContext(plan, cascadesContext);
+ Map<MarkJoinSlotReference, Pair<Boolean, Boolean>> markSlotsInfo;
+ if (conjunct.containsType(MarkJoinSlotReference.class)) {
+ List<Expression> evaluationDomain = collectEvaluationDomain(plan,
currentConjunctSubqueries);
+ evaluationDomain.addAll(extraEvaluationDomain);
+ markSlotsInfo = ExpressionUtils.inferMarkSlotNotNullMap(conjunct,
rewriteContext, evaluationDomain);
+ } else {
+ markSlotsInfo = Maps.newHashMap();
+ }
+ Map<MarkJoinSlotReference, BooleanLiteral> replaceMap =
Maps.newHashMap();
+ for (Map.Entry<MarkJoinSlotReference, Pair<Boolean, Boolean>> entry :
markSlotsInfo.entrySet()) {
+ if (entry.getValue().second) {
+ replaceMap.put(entry.getKey(), BooleanLiteral.TRUE);
+ }
+ }
+ if (!replaceMap.isEmpty()) {
+ conjunct = ExpressionUtils.replace(conjunct, replaceMap);
+ }
+ return Pair.of(conjunct, markSlotsInfo);
+ }
+
+ /*
+ * collect the complete evaluation domain of the mark slot inference: the
containing
+ * conjunct set of the filter/join, plus all the expressions inside the
correlated
+ * subquery plans. a sensitive expression (e.g. assert_true) does not need
to be inside
+ * the current conjunct: it may be a sibling conjunct of the same
filter/join, or live in
+ * a later subquery plan whose input rows are pruned together with the
outer rows when an
+ * earlier mark join is eliminated. pair.second is only safe when every
such expression
+ * is still evaluated on the same rows after the elimination, so they all
belong to the
+ * evaluation domain that the pair.second proof must be validated against.
a generated
+ * assert_true(count(*) <= 1) for a later correlated scalar subquery is
not visible here
+ * (it is synthesized by addApply after the collection), so the callers
add it separately
+ * via collectGeneratedAssertionsOfLaterConjuncts.
+ *
+ * the current conjunct's OWN subquery plans are excluded: they are
evaluated identically
+ * whether the mark join is kept or eliminated (the apply and the
resulting semi/anti
+ * join both evaluate the inner plan per outer row, or once for
uncorrelated; only the
+ * output row set differs), so a NoneMovableFunction/volatile inside them
cannot be
+ * affected by the elimination and fencing pair.second on it would only
lose valid
+ * eliminations.
+ * e.g. `ifnull(k in (select ... where assert_true(inner)), false)`: the
assert_true is
+ * inside the current conjunct's own subquery plan, which is evaluated
identically by the
+ * apply and by the resulting semi join, so eliminating the mark join
cannot suppress it;
+ * before the exclusion the mark join was kept (isMarkJoin=true) purely
because of that
+ * unreachable assert_true.
+ */
+ private List<Expression> collectEvaluationDomain(Plan plan,
Set<SubqueryExpr> currentConjunctSubqueries) {
+ List<Expression> evaluationDomain = new ArrayList<>();
+ if (plan instanceof LogicalFilter) {
+ evaluationDomain.addAll(((LogicalFilter<? extends Plan>)
plan).getConjuncts());
+ } else if (plan instanceof LogicalJoin) {
+ evaluationDomain.addAll(((LogicalJoin<?, ?>)
plan).getExpressions());
+ }
+ List<Expression> subqueryPlanExpressions = new ArrayList<>();
+ for (Expression expression : evaluationDomain) {
+ Set<SubqueryExpr> subqueries =
expression.collect(SubqueryExpr.class::isInstance);
Review Comment:
[P2] Filter plan expressions by downstream Apply reachability
Conversely, this scans every non-current subquery plan without considering
Apply order or join child. An earlier filter Apply is already below a later
target, and an opposite-side join Apply is in an independent subtree;
eliminating the target cannot skip either one's `AssertTrue`/volatile
expression. Collecting those expressions still clears Pair.second and retains
the marker-producing Apply plus its join-reordering, runtime-filter,
selectivity, and exploration barriers. Please include only plans that are
actually downstream of the target: later Applies on the filter chain, or later
Applies on the same physical join child. This is distinct from the existing
generated-assertion over-fence because these are expressions already present in
the original subquery plans.
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java:
##########
@@ -503,6 +540,199 @@ private Pair<LogicalPlan, Optional<Expression>>
addApply(SubqueryExpr subquery,
return Pair.of(logicalProject, newCorrelatedOuterExpr);
}
+ /**
+ * simplify the conjunct that contains mark join slots and infer the
behavior of each
+ * mark join slot, return the rewritten conjunct together with the mark
slots info.
+ *
+ * for each mark slot, the pair in the returned map has:
+ * Pair.first: whether the null and false values of the mark slot are
indistinguishable,
+ * i.e. the mark slot can be treated as a non-nullable
boolean. it only affects
+ * how the mark value is computed (treating null as false) and
never changes the
+ * number of output rows, so it's safe for every join type and
the filter.
+ * Pair.second: whether the original mark join can be directly eliminated
and turned into a
+ * plain semi join. a plain semi join only outputs the
matched rows, while a
+ * mark join keeps all original rows and adds a mark column,
so eliminating the
+ * mark join is only safe when discarding the unmatched rows
is already part of
+ * the containing join's semantics (inner, cross and semi
joins).
+ *
+ * when Pair.second is true, the mark slot is replaced by the true literal
in the returned
+ * conjunct, and the caller can drop the mark join slot to turn the mark
join into a plain
+ * semi join.
+ *
+ * extraEvaluationDomain extends the evaluation domain with expressions
that are not yet
+ * part of the plan but will be evaluated on the same rows later, e.g. the
generated
+ * assert_true(count(*) <= 1) that addApply synthesizes for a later
correlated scalar
+ * subquery; see collectGeneratedAssertionsOfLaterConjuncts.
+ *
+ * currentConjunctSubqueries are the subqueries of the conjunct being
processed: their own
+ * query plans are evaluated identically whether the mark join is kept or
eliminated (both
+ * the apply and the resulting semi/anti join evaluate the inner plan,
only the output row
+ * set differs), so a sensitive expression inside them cannot be affected
by the
+ * elimination and must be excluded from the evaluation domain; see
collectEvaluationDomain.
+ */
+ private Pair<Expression, Map<MarkJoinSlotReference, Pair<Boolean,
Boolean>>> simplifyConjunctWithMarkJoinSlot(
+ Expression conjunct, Plan plan, CascadesContext cascadesContext,
+ List<Expression> extraEvaluationDomain, Set<SubqueryExpr>
currentConjunctSubqueries) {
+ ExpressionRewriteContext rewriteContext = new
ExpressionRewriteContext(plan, cascadesContext);
+ Map<MarkJoinSlotReference, Pair<Boolean, Boolean>> markSlotsInfo;
+ if (conjunct.containsType(MarkJoinSlotReference.class)) {
+ List<Expression> evaluationDomain = collectEvaluationDomain(plan,
currentConjunctSubqueries);
+ evaluationDomain.addAll(extraEvaluationDomain);
+ markSlotsInfo = ExpressionUtils.inferMarkSlotNotNullMap(conjunct,
rewriteContext, evaluationDomain);
+ } else {
+ markSlotsInfo = Maps.newHashMap();
+ }
+ Map<MarkJoinSlotReference, BooleanLiteral> replaceMap =
Maps.newHashMap();
+ for (Map.Entry<MarkJoinSlotReference, Pair<Boolean, Boolean>> entry :
markSlotsInfo.entrySet()) {
+ if (entry.getValue().second) {
+ replaceMap.put(entry.getKey(), BooleanLiteral.TRUE);
+ }
+ }
+ if (!replaceMap.isEmpty()) {
+ conjunct = ExpressionUtils.replace(conjunct, replaceMap);
+ }
+ return Pair.of(conjunct, markSlotsInfo);
+ }
+
+ /*
+ * collect the complete evaluation domain of the mark slot inference: the
containing
+ * conjunct set of the filter/join, plus all the expressions inside the
correlated
+ * subquery plans. a sensitive expression (e.g. assert_true) does not need
to be inside
+ * the current conjunct: it may be a sibling conjunct of the same
filter/join, or live in
+ * a later subquery plan whose input rows are pruned together with the
outer rows when an
+ * earlier mark join is eliminated. pair.second is only safe when every
such expression
+ * is still evaluated on the same rows after the elimination, so they all
belong to the
+ * evaluation domain that the pair.second proof must be validated against.
a generated
+ * assert_true(count(*) <= 1) for a later correlated scalar subquery is
not visible here
+ * (it is synthesized by addApply after the collection), so the callers
add it separately
+ * via collectGeneratedAssertionsOfLaterConjuncts.
+ *
+ * the current conjunct's OWN subquery plans are excluded: they are
evaluated identically
+ * whether the mark join is kept or eliminated (the apply and the
resulting semi/anti
+ * join both evaluate the inner plan per outer row, or once for
uncorrelated; only the
+ * output row set differs), so a NoneMovableFunction/volatile inside them
cannot be
+ * affected by the elimination and fencing pair.second on it would only
lose valid
+ * eliminations.
+ * e.g. `ifnull(k in (select ... where assert_true(inner)), false)`: the
assert_true is
+ * inside the current conjunct's own subquery plan, which is evaluated
identically by the
+ * apply and by the resulting semi join, so eliminating the mark join
cannot suppress it;
+ * before the exclusion the mark join was kept (isMarkJoin=true) purely
because of that
+ * unreachable assert_true.
+ */
+ private List<Expression> collectEvaluationDomain(Plan plan,
Set<SubqueryExpr> currentConjunctSubqueries) {
+ List<Expression> evaluationDomain = new ArrayList<>();
+ if (plan instanceof LogicalFilter) {
+ evaluationDomain.addAll(((LogicalFilter<? extends Plan>)
plan).getConjuncts());
+ } else if (plan instanceof LogicalJoin) {
+ evaluationDomain.addAll(((LogicalJoin<?, ?>)
plan).getExpressions());
+ }
+ List<Expression> subqueryPlanExpressions = new ArrayList<>();
+ for (Expression expression : evaluationDomain) {
+ Set<SubqueryExpr> subqueries =
expression.collect(SubqueryExpr.class::isInstance);
+ for (SubqueryExpr subquery : subqueries) {
+ // skip the current conjunct's own subquery plans: see the
domain doc above
+ if (currentConjunctSubqueries.contains(subquery)) {
+ continue;
+ }
+ collectPlanExpressions(subquery.getQueryPlan(),
subqueryPlanExpressions);
+ }
+ }
+ evaluationDomain.addAll(subqueryPlanExpressions);
+ return evaluationDomain;
+ }
+
+ private void collectPlanExpressions(Plan plan, List<Expression>
expressions) {
+ expressions.addAll(plan.getExpressions());
+ for (Plan child : plan.children()) {
+ collectPlanExpressions(child, expressions);
+ }
+ }
+
+ /*
+ * whether addApply will synthesize the runtime assert_true(count(*) <= 1)
for the
+ * subquery: a correlated scalar subquery without a top-level scalar agg
that is not
+ * limit-one-eliminated. a top-level scalar agg returns at most one row
and a
+ * limit-one-eliminated subquery is guaranteed to produce at most one row,
so no check
+ * is generated for them. the check references a count slot that only
exists after
+ * addApply, so it is invisible to collectEvaluationDomain and a preceding
mark join
+ * whose elimination prunes the rows reaching the check must be fenced.
+ */
+ private static boolean isCorrelatedScalarNeedingRuntimeCheck(SubqueryExpr
subquery) {
+ if (!(subquery instanceof ScalarSubquery)) {
+ return false;
+ }
+ ScalarSubquery scalar = (ScalarSubquery) subquery;
+ return !scalar.getCorrelateSlots().isEmpty()
+ && !scalar.hasTopLevelScalarAgg()
+ && !scalar.limitOneIsEliminated();
+ }
+
+ /*
+ * collect a representative of the runtime assert_true(count(*) <= 1) that
addApply
+ * will synthesize for every correlated scalar subquery in the conjuncts
after
+ * currentIndex: those applies are built above the current conjunct's
apply, so
+ * eliminating the current conjunct's mark join would prune the rows that
reach the
+ * generated assertion and suppress its error. only the sensitive-function
type
+ * matters for the inference fence, so a representative assertion with a
fresh count
+ * slot is enough to fence the elimination.
+ *
+ * whether a later conjunct's generated assertion is affected by the
current mark join
+ * elimination depends on the plan shape, which differs between the two
paths:
+ *
+ * filter path (FILTER_SUBQUERY_TO_APPLY, relatedInfoList == null): every
conjunct's
+ * apply is stacked on ONE chain above the filter's child (each new apply
wraps the
+ * accumulated plan), so for
+ * `t1 join t2 on xx where t1.k in (sub1) and t2.b = (scalar2)`
+ * the plan becomes
+ * project(assert_true(count<=1)) <- scalar2's generated
assertion
+ * +-- Apply1(t2.b = scalar2) <- wraps Apply0
+ * +-- Apply0(t1.k in sub1) <- mark join, wraps the
whole join
+ * +-- Join(t1, t2)
+ * even though sub1 correlates to t1 and scalar2 to t2, both applies sit
ABOVE the whole
+ * join on the same chain: the joined rows already carry both sides'
columns, and
+ * eliminating Apply0's mark join prunes joined rows BEFORE Apply1
evaluates, so the
+ * assertion of every later conjunct is downstream of every earlier mark
join. hence all
+ * later conjuncts must be fenced here.
+ *
+ * join path (JOIN_SUBQUERY_TO_APPLY, relatedInfoList != null): each
conjunct's apply is
+ * attached to the LEFT or RIGHT child of the join (collectRelatedInfo
decides the side),
+ * so for
+ * `t1 join t2 on t1.k in (sub1) and t2.b = (scalar2)`
+ * the plan becomes
+ * Join(other=[M1, t2.b = scalar2_out])
+ * +-- Apply0(t1.k in sub1) <- left subtree, mark join
+ * | +-- t1
+ * +-- project(assert_true(count<=1)) <- right subtree, scalar2's
assertion
+ * +-- Apply1(t2.b = scalar2)
+ * +-- t2
+ * the left and right subtrees are evaluated independently BEFORE the join
combines them,
+ * so eliminating the current mark join only prunes rows on its OWN side:
an OPPOSITE-side
+ * later conjunct is unaffected and must NOT be fenced (the pre-fix code
fenced it, losing
+ * a valid elimination), while a SAME-side later conjunct is stacked above
the current
+ * apply and IS fenced.
+ */
+ private List<Expression> collectGeneratedAssertionsOfLaterConjuncts(
+ int currentIndex, List<Set<SubqueryExpr>> subqueryExprsList,
+ List<RelatedInfo> relatedInfoList) {
+ List<Expression> generatedAssertions = new ArrayList<>();
+ for (int j = currentIndex + 1; j < subqueryExprsList.size(); ++j) {
+ if (relatedInfoList != null && relatedInfoList.get(j) !=
relatedInfoList.get(currentIndex)) {
Review Comment:
[P1] Compare the selected Apply child rather than `RelatedInfo` identity
`Unrelated` and `RelatedToRight` compare unequal here, but the construction
above sends both through the right-child branch. For inner-join ON conjuncts
ordered as an uncorrelated `EXISTS (SELECT 1 FROM empty_e)` followed by an
output-used right-correlated scalar, the first marker is removable and this
check skips the later scalar assertion. The marker-free EXISTS becomes a cross
join with `Limit(1, empty_e)`, removes every right row below the scalar Apply,
and suppresses a duplicate-group cardinality error; the retained mark form
preserves those rows with a false marker and raises. Please compare effective
physical sides (only `RelatedToLeft` maps left; `Unrelated` and
`RelatedToRight` map right) and add this expected-error regression.
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java:
##########
@@ -503,6 +540,199 @@ private Pair<LogicalPlan, Optional<Expression>>
addApply(SubqueryExpr subquery,
return Pair.of(logicalProject, newCorrelatedOuterExpr);
}
+ /**
+ * simplify the conjunct that contains mark join slots and infer the
behavior of each
+ * mark join slot, return the rewritten conjunct together with the mark
slots info.
+ *
+ * for each mark slot, the pair in the returned map has:
+ * Pair.first: whether the null and false values of the mark slot are
indistinguishable,
+ * i.e. the mark slot can be treated as a non-nullable
boolean. it only affects
+ * how the mark value is computed (treating null as false) and
never changes the
+ * number of output rows, so it's safe for every join type and
the filter.
+ * Pair.second: whether the original mark join can be directly eliminated
and turned into a
+ * plain semi join. a plain semi join only outputs the
matched rows, while a
+ * mark join keeps all original rows and adds a mark column,
so eliminating the
+ * mark join is only safe when discarding the unmatched rows
is already part of
+ * the containing join's semantics (inner, cross and semi
joins).
+ *
+ * when Pair.second is true, the mark slot is replaced by the true literal
in the returned
+ * conjunct, and the caller can drop the mark join slot to turn the mark
join into a plain
+ * semi join.
+ *
+ * extraEvaluationDomain extends the evaluation domain with expressions
that are not yet
+ * part of the plan but will be evaluated on the same rows later, e.g. the
generated
+ * assert_true(count(*) <= 1) that addApply synthesizes for a later
correlated scalar
+ * subquery; see collectGeneratedAssertionsOfLaterConjuncts.
+ *
+ * currentConjunctSubqueries are the subqueries of the conjunct being
processed: their own
+ * query plans are evaluated identically whether the mark join is kept or
eliminated (both
+ * the apply and the resulting semi/anti join evaluate the inner plan,
only the output row
+ * set differs), so a sensitive expression inside them cannot be affected
by the
+ * elimination and must be excluded from the evaluation domain; see
collectEvaluationDomain.
+ */
+ private Pair<Expression, Map<MarkJoinSlotReference, Pair<Boolean,
Boolean>>> simplifyConjunctWithMarkJoinSlot(
+ Expression conjunct, Plan plan, CascadesContext cascadesContext,
+ List<Expression> extraEvaluationDomain, Set<SubqueryExpr>
currentConjunctSubqueries) {
+ ExpressionRewriteContext rewriteContext = new
ExpressionRewriteContext(plan, cascadesContext);
+ Map<MarkJoinSlotReference, Pair<Boolean, Boolean>> markSlotsInfo;
+ if (conjunct.containsType(MarkJoinSlotReference.class)) {
+ List<Expression> evaluationDomain = collectEvaluationDomain(plan,
currentConjunctSubqueries);
+ evaluationDomain.addAll(extraEvaluationDomain);
+ markSlotsInfo = ExpressionUtils.inferMarkSlotNotNullMap(conjunct,
rewriteContext, evaluationDomain);
+ } else {
+ markSlotsInfo = Maps.newHashMap();
+ }
+ Map<MarkJoinSlotReference, BooleanLiteral> replaceMap =
Maps.newHashMap();
+ for (Map.Entry<MarkJoinSlotReference, Pair<Boolean, Boolean>> entry :
markSlotsInfo.entrySet()) {
+ if (entry.getValue().second) {
+ replaceMap.put(entry.getKey(), BooleanLiteral.TRUE);
+ }
+ }
+ if (!replaceMap.isEmpty()) {
+ conjunct = ExpressionUtils.replace(conjunct, replaceMap);
+ }
+ return Pair.of(conjunct, markSlotsInfo);
+ }
+
+ /*
+ * collect the complete evaluation domain of the mark slot inference: the
containing
+ * conjunct set of the filter/join, plus all the expressions inside the
correlated
+ * subquery plans. a sensitive expression (e.g. assert_true) does not need
to be inside
+ * the current conjunct: it may be a sibling conjunct of the same
filter/join, or live in
+ * a later subquery plan whose input rows are pruned together with the
outer rows when an
+ * earlier mark join is eliminated. pair.second is only safe when every
such expression
+ * is still evaluated on the same rows after the elimination, so they all
belong to the
+ * evaluation domain that the pair.second proof must be validated against.
a generated
+ * assert_true(count(*) <= 1) for a later correlated scalar subquery is
not visible here
+ * (it is synthesized by addApply after the collection), so the callers
add it separately
+ * via collectGeneratedAssertionsOfLaterConjuncts.
+ *
+ * the current conjunct's OWN subquery plans are excluded: they are
evaluated identically
+ * whether the mark join is kept or eliminated (the apply and the
resulting semi/anti
+ * join both evaluate the inner plan per outer row, or once for
uncorrelated; only the
+ * output row set differs), so a NoneMovableFunction/volatile inside them
cannot be
+ * affected by the elimination and fencing pair.second on it would only
lose valid
+ * eliminations.
+ * e.g. `ifnull(k in (select ... where assert_true(inner)), false)`: the
assert_true is
+ * inside the current conjunct's own subquery plan, which is evaluated
identically by the
+ * apply and by the resulting semi join, so eliminating the mark join
cannot suppress it;
+ * before the exclusion the mark join was kept (isMarkJoin=true) purely
because of that
+ * unreachable assert_true.
+ */
+ private List<Expression> collectEvaluationDomain(Plan plan,
Set<SubqueryExpr> currentConjunctSubqueries) {
+ List<Expression> evaluationDomain = new ArrayList<>();
+ if (plan instanceof LogicalFilter) {
+ evaluationDomain.addAll(((LogicalFilter<? extends Plan>)
plan).getConjuncts());
+ } else if (plan instanceof LogicalJoin) {
+ evaluationDomain.addAll(((LogicalJoin<?, ?>)
plan).getExpressions());
+ }
+ List<Expression> subqueryPlanExpressions = new ArrayList<>();
+ for (Expression expression : evaluationDomain) {
+ Set<SubqueryExpr> subqueries =
expression.collect(SubqueryExpr.class::isInstance);
+ for (SubqueryExpr subquery : subqueries) {
+ // skip the current conjunct's own subquery plans: see the
domain doc above
+ if (currentConjunctSubqueries.contains(subquery)) {
Review Comment:
[P1] Include later same-conjunct Applies in this target's safety domain
This excludes every subquery plan in the current conjunct, while
`collectGeneratedAssertionsOfLaterConjuncts` starts only at the next conjunct
index. In a single filter conjunct such as `nvl(nvl(o.k IN (...), false) AND
o.x = (SELECT s.v ... WHERE s.g = o.g), false)`, preorder builds the IN Apply
below the scalar Apply and Pair.second is true for the IN marker. Removing it
creates a left-semi join that can discard the only outer row whose scalar group
has multiple results before the later `Count`/`AssertTrue` runs, so the query
returns instead of raising `correlate scalar subquery must return only 1 row`.
The existing later-conjunct fence does not cover this same-index shape. Please
make the domain target/order-specific: exclude the target and already-lower
Applies, but include subsequent same-conjunct sensitive plans and output-used
generated scalar assertions, with an expected-error regression.
##########
regression-test/suites/query_p0/subquery/subquery_unnesting.groovy:
##########
@@ -145,4 +146,294 @@ suite ("subquery_unnesting") {
FROM (SELECT 1 AS x) t
WHERE 1 NOT IN (SELECT CAST(NULL AS INT));
"""
+
+ // =====================================================================
+ // mark join elimination in the join ON condition.
+ //
+ // inferMarkSlotNotNullMap returns a pair for each mark join slot:
+ // Pair.first : the null and false values of the mark slot are
+ // indistinguishable, so the mark slot can be treated as a
+ // non-nullable boolean (null is computed as false when
+ // producing the mark value). this never changes the number
+ // of output rows, so it's safe for all join types.
+ // Pair.second: the original mark join can be directly eliminated and
+ // turned into a plain semi join. a plain semi join only
+ // outputs the matched rows, while a mark join keeps all
+ // original rows plus the mark column, so eliminating the
+ // mark join is only safe for inner, cross and semi joins
+ // where dropping the unmatched rows is already part of the
+ // join semantics.
+ //
+ // take the query below as an example:
+ // select t1.* from t1 left join t2 on t1.k2 = t2.k3
+ // and t1.k1 in (select t3.k1 from t3 where t1.k2 = t3.k2)
+ // for the outer join the mark join must be kept: the unmatched left rows
+ // (mark = false/null) must be preserved with null columns of t2, while a
+ // plain semi join would drop them. so the analyzed plan must keep
+ // isMarkJoin=true and only infer the non-nullable mark
+ // (isMarkJoinSlotNotNull=true). for inner/cross/semi join the unmatched
+ // rows are dropped anyway, so the mark join can be safely eliminated
+ // (isMarkJoin=false and the mark slot is replaced by the true literal).
+ //
+ // note: anti join also keeps the mark join for the null-aware semantics
+ // of NOT IN, but executing an anti join with a subquery in its ON
+ // condition is a pre-existing unsupported path in physical planning, so
+ // only the analyzed plan is checked here. asof join's ON clause only
+ // allows equal conjuncts, so a subquery can never appear in it and the
+ // asof branch in the code is defensive only.
+ // =====================================================================
+
+ // inner join: the mark join is eliminated (isMarkJoin=false,
MarkJoinSlotReference=empty)
+ explain {
+ sql("""analyzed plan select t1.* from t1 inner join t2 on t1.k2 = t2.k3
+ and t1.k1 in (select t3.k1 from t3 where t1.k2 = t3.k2) order
by t1.k1, t1.k2;""")
+ contains("isMarkJoin=false")
+ contains("MarkJoinSlotReference=empty")
+ }
+ // semi join: the mark join is eliminated too
+ explain {
+ sql("""analyzed plan select t1.* from t1 left semi join t2 on t1.k2 =
t2.k3
+ and t1.k1 in (select t3.k1 from t3 where t1.k2 = t3.k2) order
by t1.k1, t1.k2;""")
+ contains("isMarkJoin=false")
+ contains("MarkJoinSlotReference=empty")
+ }
+ // outer join: the mark join must be kept, and the mark slot's null/false
+ // equivalence (isMarkJoinSlotNotNull=true) is still inferred
+ explain {
+ sql("""analyzed plan select t1.* from t1 left join t2 on t1.k2 = t2.k3
+ and t1.k1 in (select t3.k1 from t3 where t1.k2 = t3.k2) order
by t1.k1, t1.k2;""")
+ contains("isMarkJoin=true")
+ contains("isMarkJoinSlotNotNull=true")
+ }
+ // anti join: the mark join must be kept for the null-aware semantics of
NOT IN
+ explain {
+ sql("""analyzed plan select t1.* from t1 left anti join t2 on t1.k2 =
t2.k3
+ and t1.k1 not in (select t3.k1 from t3 where t1.k2 = t3.k2)
order by t1.k1, t1.k2;""")
+ contains("isMarkJoin=true")
+ contains("isMarkJoinSlotNotNull=true")
+ }
+
+ // result checks: the mark join elimination must not change the query
results.
+ // (the outer join results with subquery in the ON condition are already
+ // covered by qt_select37 / qt_select43, the mark join is kept there)
+ // inner join with IN subquery in the ON condition (mark join eliminated)
+ qt_select66 """select t1.* from t1 inner join t2 on t1.k2 = t2.k3 and
t1.k1 in (select t3.k1 from t3 where t1.k2 = t3.k2) order by t1.k1, t1.k2;"""
+ // inner join with NOT IN subquery in the ON condition (mark join
eliminated)
+ qt_select67 """select t1.* from t1 inner join t2 on t1.k2 = t2.k3 and
t1.k1 not in (select t3.k1 from t3 where t1.k2 = t3.k2) order by t1.k1,
t1.k2;"""
+ // left semi join with IN subquery in the ON condition (mark join
eliminated)
+ qt_select68 """select t1.* from t1 left semi join t2 on t1.k2 = t2.k3 and
t1.k1 in (select t3.k1 from t3 where t1.k2 = t3.k2) order by t1.k1, t1.k2;"""
+
+ // error-behavior regression: the mark join must NOT be eliminated when
the filter
+ // contains assert_true (a NoneMovableFunction). although M = false and M
= null both
+ // fold the predicate to false (the row-truth proof), eliminating the mark
join changes
+ // which rows reach assert_true: the semi join prunes the unmatched rows
before the
+ // filter, so assert_true is no longer evaluated on them and its error is
suppressed.
+ // with the mark join kept, all rows reach the filter and assert_true
throws on the
+ // unmatched guard = false rows.
+ // data: M = assert_t.k1 in (assert_s.k1 where assert_s.k2 = assert_t.k2),
so only
+ // row (2,2) matches; guard = assert_t.k2 = 2 is false exactly on the
unmatched rows
+ // (1,1) and (3,3)
+ sql "drop table if exists assert_t"
+ sql "drop table if exists assert_s"
+ sql """create table assert_t (k1 bigint, k2 bigint) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """create table assert_s (k1 bigint, k2 bigint) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """insert into assert_t values (1,1),(2,2),(3,3);"""
+ sql """insert into assert_s values (2,2);"""
+ test {
+ sql """select assert_t.k1 from assert_t
+ where ifnull(
+ ifnull(assert_t.k1 in (select assert_s.k1 from assert_s
+ where assert_s.k2 = assert_t.k2), false)
+ and assert_true(assert_t.k2 = 2, 'assert failed'),
+ false);"""
+ exception "assert failed"
+ }
+
+ // error-behavior regressions for the "complete evaluation domain" fence:
the mark join
+ // must not be eliminated when a NoneMovableFunction (assert_true) exists
anywhere in the
+ // affected evaluation domain, even if it is NOT inside the mark conjunct
itself.
+ sql "drop table if exists assert_u"
+ sql """create table assert_u (k1 bigint, k2 bigint) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """insert into assert_u values (1,1),(2,2),(3,3);"""
+
+ // sibling conjunct: assert_true is a SIBLING conjunct of the eliminable
mark conjunct.
+ // the mark conjunct must be a MARKER-REQUIRING form (ifnull(k1 in (...),
false)): a bare
+ // `k1 not in (...)` is itself a top-level SubqueryExpr, so
shouldOutputMarkJoinSlot returns
+ // false and no mark slot is created (it becomes a plain left anti join
that emits the
+ // unmatched rows anyway, so the test would stay green even without the
fence). with the
+ // mark form the analyzed plan must keep isMarkJoin=true, which is the
sensitive signal:
+ // removing the complete evaluation domain fence turns it into
isMarkJoin=false (the mark
+ // join is eliminated into a semi join). note that the error behavior is
NOT the sensitive
+ // signal here — assert_true(k2 = 2) only references the outer columns, so
the optimizer
+ // pushes it below the join (into the outer scan) and it raises the error
on every outer
+ // row regardless of the elimination.
+ explain {
+ sql("""analyzed plan select assert_t.k1 from assert_t
+ where ifnull(assert_t.k1 in (select assert_s.k1 from assert_s
+ where assert_s.k2 = assert_t.k2), false)
+ and assert_true(assert_t.k2 = 2, 'assert failed');""")
+ contains("isMarkJoin=true")
+ }
+
+ // sensitive expression inside a later subquery plan: assert_true lives in
the filter of
+ // a LATER subquery (the EXISTS one). marker replacement erases that plan
from the
+ // earlier IN conjunct, so the complete evaluation domain (including all
subquery plans)
+ // must fence the earlier IN apply from being eliminated into a semi join.
the 'assert
+ // failed' error must still be raised.
+ test {
+ sql """select assert_t.k1 from assert_t
+ where ifnull(assert_t.k1 in (select assert_s.k1 from assert_s
+ where assert_s.k2 = assert_t.k2), false)
+ and exists (select 1 from assert_u
+ where assert_u.k2 = assert_t.k2
+ and assert_true(assert_u.k1 = 2, 'assert
failed'));"""
+ exception "assert failed"
+ }
+
+ // error-behavior regression for the retained-mark non-nullable inference
(pair.first):
+ // ((M and assert_true(guard, 'bad')) or flag) keeps the mark join
(pair.second = false)
+ // and pair.first alone would mark M non-nullable. M = k1 in (select null)
is NULL for
+ // every row (null in the build side), and treating that null as false
changes how
+ // assert_true is evaluated: the vectorized AND evaluates its right
operand for a
+ // nullable null input but can return early for an all-false non-null
column, so the
+ // required 'assert failed' error would be suppressed. pair.first must be
fenced to
+ // false so M stays null and assert_true is evaluated on every row.
+ sql "drop table if exists null_src"
+ sql """create table null_src (v bigint null) DUPLICATE KEY(v)
+ DISTRIBUTED BY HASH(v) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """insert into null_src values (null);"""
+ sql "drop table if exists guard_t"
+ sql """create table guard_t (k1 bigint, guard bigint) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """insert into guard_t values (1,0),(2,0),(3,0);"""
+ test {
+ sql """select guard_t.k1 from guard_t
+ where (guard_t.k1 in (select null from null_src)
+ and assert_true(guard_t.guard = 1, 'assert failed'))
+ or guard_t.guard = 2;"""
+ exception "assert failed"
+ }
+
+ // =====================================================================
+ // split-fence regression: for an uncorrelated NULLABLE positive IN in a
join ON
+ // condition with a sensitive SIBLING expression, pair.first
(isMarkJoinSlotNotNull)
+ // must be kept while pair.second (isMarkJoin=true) is fenced.
+ //
+ // the mark predicate `uncor_in_t1.k in (select c from uncor_in_t3)` is
itself CLEAN
+ // (no assert_true inside it), so the current-predicate fence does not
apply to it.
+ // the sibling assert_true only lives in the evaluation domain, so it
fences pair.second
+ // only: the mark join is kept (isMarkJoin=true) because eliminating it
would prune the
+ // unmatched rows before assert_true, but pair.first stays true because
the sibling
+ // cannot observe this generated marker's null-vs-false mapping
(pair.first keeps the
+ // apply and only maps the marker's null to false).
+ //
+ // this matters beyond the analyzed plan: with isMarkJoinSlotNotNull=true,
InApplyToJoin
+ // moves the (nullable) IN equality into the hash conjuncts, so
JoinUtils.couldShuffle
+ // stays true and the physical planner keeps the shuffle alternative. if
pair.first were
+ // wrongly fenced (isMarkJoinSlotNotNull=false), the equality would stay
in the
+ // markConjuncts only, producing a standalone mark join with no hash
conjuncts, which
+ // couldShuffle forces to broadcast. the uncorrelated IN (no correlation
hash conjunct)
+ // makes this mark join the only join deciding the distribution, so the
regression pins
+ // the isMarkJoinSlotNotNull signal directly.
+ sql "drop table if exists uncor_in_t1"
+ sql "drop table if exists uncor_in_t2"
+ sql "drop table if exists uncor_in_t3"
+ sql """create table uncor_in_t1 (k bigint, a bigint) DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """create table uncor_in_t2 (b bigint) DUPLICATE KEY(b)
+ DISTRIBUTED BY HASH(b) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """create table uncor_in_t3 (c bigint null) DUPLICATE KEY(c)
+ DISTRIBUTED BY HASH(c) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """insert into uncor_in_t1 values (1,1),(2,2),(3,3);"""
+ sql """insert into uncor_in_t2 values (1),(2);"""
+ sql """insert into uncor_in_t3 values (1),(null);"""
+ explain {
+ sql("""analyzed plan select uncor_in_t1.* from uncor_in_t1
+ join uncor_in_t2 on uncor_in_t1.a = uncor_in_t2.b
+ and uncor_in_t1.k in (select c from uncor_in_t3)
+ and assert_true(uncor_in_t1.k > 0, 'assert failed')
+ order by uncor_in_t1.k;""")
+ contains("isMarkJoin=true")
+ contains("isMarkJoinSlotNotNull=true")
+ }
+
+ // =====================================================================
+ // current-conjunct-own-subquery exclusion regression: a
NoneMovableFunction inside
+ // the CURRENT conjunct's OWN subquery plan must NOT fence the mark join
elimination.
+ //
+ // the mark predicate `ifnull(k1 in (select ...), false)` is clean, and
the assert_true
+ // lives inside the subquery's own plan (its filter). the inner plan is
evaluated
+ // identically whether the mark join is kept or eliminated: both the apply
and the
+ // resulting semi join evaluate the subquery (per outer row for a
correlated subquery),
+ // only the output row set differs, so assert_true inside it cannot be
affected by the
+ // elimination. the evaluation domain must therefore exclude the current
conjunct's own
+ // subquery plans; otherwise the mark join is kept (isMarkJoin=true)
purely because of a
+ // sensitive expression the elimination cannot reach. with the exclusion
the mark join is
+ // eliminated (isMarkJoin=false) and the subquery is still evaluated, so
assert_true
+ // still raises its error on the inner row with k2 = 0.
+ sql "drop table if exists inner_assert_t"
+ sql "drop table if exists inner_assert_s"
+ sql """create table inner_assert_t (k1 bigint, k2 bigint) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """create table inner_assert_s (k1 bigint, k2 bigint) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """insert into inner_assert_t values (1,1),(2,2),(3,3),(4,4);"""
+ sql """insert into inner_assert_s values (2,2),(4,0);"""
+ explain {
+ sql("""analyzed plan select inner_assert_t.k1 from inner_assert_t
+ where ifnull(inner_assert_t.k1 in (select inner_assert_s.k1
from inner_assert_s
+ where inner_assert_s.k1 = inner_assert_t.k1
+ and assert_true(inner_assert_s.k2 > 0, 'assert
failed')), false)
+ order by inner_assert_t.k1;""")
+ contains("isMarkJoin=false")
+ }
+ test {
+ sql """select inner_assert_t.k1 from inner_assert_t
+ where ifnull(inner_assert_t.k1 in (select inner_assert_s.k1
from inner_assert_s
+ where inner_assert_s.k1 = inner_assert_t.k1
+ and assert_true(inner_assert_s.k2 > 0, 'assert
failed')), false)
+ order by inner_assert_t.k1;"""
+ exception "assert failed"
+ }
+
+ // =====================================================================
+ // opposite-side exclusion regression: in the JOIN path, a later
correlated scalar on
+ // the OPPOSITE side of the join must NOT fence the current mark join
elimination.
+ //
+ // the join has two subquery conjuncts on opposite sides: a mark IN
correlated to the
+ // left (side_join_t.k in (select ... where side_join_s.g =
side_join_t.g)) and a later
+ // correlated scalar correlated to the right (side_join_u.b = (select
side_join_v.c ...
+ // where side_join_v.h = side_join_u.h)), which generates the runtime
+ // assert_true(count(*) <= 1) in the right subtree. the right subtree is
an independent
+ // branch: eliminating the left mark join (semi join pruning the left
rows) does not
+ // change which right rows reach the generated assertion, so the fence on
it is
+ // unnecessary and the left mark join can be eliminated (isMarkJoin=false).
+ sql "drop table if exists side_join_t"
+ sql "drop table if exists side_join_s"
+ sql "drop table if exists side_join_u"
+ sql "drop table if exists side_join_v"
+ sql """create table side_join_t (k bigint, g bigint) DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """create table side_join_s (k bigint, g bigint) DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """create table side_join_u (b bigint, h bigint) DUPLICATE KEY(b)
+ DISTRIBUTED BY HASH(b) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """create table side_join_v (c bigint, h bigint) DUPLICATE KEY(c)
+ DISTRIBUTED BY HASH(c) BUCKETS 1
PROPERTIES('replication_num'='1');"""
+ sql """insert into side_join_t values (1,1),(2,2);"""
+ sql """insert into side_join_s values (1,1),(2,2),(3,3);"""
+ sql """insert into side_join_u values (10,10),(20,20);"""
+ sql """insert into side_join_v values (100,10),(200,20);"""
+ explain {
+ sql("""analyzed plan select side_join_t.k, side_join_u.b from
side_join_t
+ join side_join_u on side_join_t.k in (select side_join_s.k
from side_join_s
+ where side_join_s.g = side_join_t.g)
+ and side_join_u.b = (select side_join_v.c from side_join_v
+ where side_join_v.h = side_join_u.h)
+ order by side_join_t.k;""")
+ contains("isMarkJoin=false")
Review Comment:
[P2] Make this assertion target the IN Apply
This query has both the target left-side IN Apply and a right-side scalar
Apply. The scalar Apply never has a marker, so it independently prints
`isMarkJoin=false`. A regression that incorrectly retains the IN marker
therefore contains both `isMarkJoin=true` and `isMarkJoin=false`, and this
positive substring check still passes. Please also assert
`notContains("isMarkJoin=true")` or match the exact target Apply shape so the
test fails when the intended elimination is lost.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
