sunchao commented on code in PR #25053:
URL: https://github.com/apache/datafusion/pull/25053#discussion_r3975301471
##########
datafusion/physical-expr/src/equivalence/properties/joins.rs:
##########
@@ -107,6 +91,96 @@ pub fn join_equivalence_properties(
Ok(result)
}
+/// Append build orderings only when equal probe ordering values identify at
most
+/// one build row. The suffix is then constant within each probe ordering
group,
+/// even if probe rows repeat. For an outer join preserving the probe side, all
+/// join keys must be fixed within the group, and the caller must rule out
filters,
+/// so the group cannot mix matched build rows with NULL-extended rows.
+fn unique_build_join_orderings(
+ probe: &EquivalenceProperties,
+ build: &EquivalenceProperties,
+ on: &[(PhysicalExprRef, PhysicalExprRef)],
+ probe_side: JoinSide,
+ preserves_unmatched_probe: bool,
+) -> Result<OrderingEquivalenceClass> {
+ if build.constraints().is_empty() || build.oeq_class().is_empty() {
+ return Ok(OrderingEquivalenceClass::default());
+ }
+ let on = on
+ .iter()
+ .map(|(left, right)| {
+ let (probe_key, build_key) = match probe_side {
+ JoinSide::Left => (left, right),
+ JoinSide::Right => (right, left),
+ JoinSide::None => unreachable!(),
+ };
+ (
+ probe.eq_group().normalize_expr(Arc::clone(probe_key)),
+ build.eq_group().normalize_expr(Arc::clone(build_key)),
+ )
+ })
+ .collect::<Vec<_>>();
+ let mut valid_orderings = Vec::new();
+ for ordering in probe.oeq_class().iter() {
+ let probe_exprs = ordering
+ .iter()
+ .map(|sort|
probe.eq_group().normalize_expr(Arc::clone(&sort.expr)))
+ .collect::<Vec<_>>();
+
+ // Outer joins must have the same match status throughout the group.
+ if preserves_unmatched_probe
+ && !on
+ .iter()
+ .all(|(probe_key, _)| probe_exprs.contains(probe_key))
+ {
+ continue;
+ }
+ if !ordering_covers_unique_build_key(build, &on, &probe_exprs) {
+ continue;
+ }
+ valid_orderings.push(ordering.clone());
+ }
+ let mut probe_orderings = OrderingEquivalenceClass::new(valid_orderings);
+ if probe_orderings.is_empty() {
+ return Ok(probe_orderings);
+ }
+ let mut build_orderings = build.oeq_class().clone();
+ match probe_side {
+ JoinSide::Left =>
build_orderings.add_offset(probe.schema.fields().len() as _)?,
+ JoinSide::Right =>
probe_orderings.add_offset(build.schema.fields().len() as _)?,
+ JoinSide::None => unreachable!(),
+ }
+ Ok(probe_orderings.join_suffix(&build_orderings))
+}
+
+/// Check whether the probe ordering determines a unique build key.
+/// Join keys and probe ordering expressions must already be normalized.
+fn ordering_covers_unique_build_key(
+ build: &EquivalenceProperties,
+ on: &[(PhysicalExprRef, PhysicalExprRef)],
+ probe_exprs: &[PhysicalExprRef],
+) -> bool {
+ build.constraints().iter().any(|constraint| {
+ let (Constraint::PrimaryKey(indices) | Constraint::Unique(indices)) =
constraint;
+ !indices.is_empty()
+ && indices.iter().all(|&index| {
+ let Some(field) = build.schema.fields().get(index) else {
+ return false;
+ };
+ // UNIQUE can contain repeated NULLs. Without null-equality
+ // information, only non-null UNIQUE columns prove uniqueness.
+ if matches!(constraint, Constraint::Unique(_)) &&
field.is_nullable() {
+ return false;
Review Comment:
[P2] Preserve nullable UNIQUE proofs for ordinary equality
With NullEqualsNothing, NULL build keys cannot match, so nullable UNIQUE(k)
still guarantees at most one matching build row. Rejecting it here introduces
PartialSortExec for ORDER BY probe.k, build.v LIMIT 1. The base returns
immediately; the updated head waits indefinitely when the probe-key group never
finishes. Finite controls pass. Include null-equality semantics in this proof.
##########
datafusion/physical-expr/src/equivalence/properties/joins.rs:
##########
@@ -107,6 +91,96 @@ pub fn join_equivalence_properties(
Ok(result)
}
+/// Append build orderings only when equal probe ordering values identify at
most
+/// one build row. The suffix is then constant within each probe ordering
group,
+/// even if probe rows repeat. For an outer join preserving the probe side, all
+/// join keys must be fixed within the group, and the caller must rule out
filters,
+/// so the group cannot mix matched build rows with NULL-extended rows.
+fn unique_build_join_orderings(
+ probe: &EquivalenceProperties,
+ build: &EquivalenceProperties,
+ on: &[(PhysicalExprRef, PhysicalExprRef)],
+ probe_side: JoinSide,
+ preserves_unmatched_probe: bool,
+) -> Result<OrderingEquivalenceClass> {
+ if build.constraints().is_empty() || build.oeq_class().is_empty() {
+ return Ok(OrderingEquivalenceClass::default());
+ }
+ let on = on
+ .iter()
+ .map(|(left, right)| {
+ let (probe_key, build_key) = match probe_side {
+ JoinSide::Left => (left, right),
+ JoinSide::Right => (right, left),
+ JoinSide::None => unreachable!(),
+ };
+ (
+ probe.eq_group().normalize_expr(Arc::clone(probe_key)),
+ build.eq_group().normalize_expr(Arc::clone(build_key)),
+ )
+ })
+ .collect::<Vec<_>>();
+ let mut valid_orderings = Vec::new();
+ for ordering in probe.oeq_class().iter() {
+ let probe_exprs = ordering
+ .iter()
+ .map(|sort|
probe.eq_group().normalize_expr(Arc::clone(&sort.expr)))
+ .collect::<Vec<_>>();
+
+ // Outer joins must have the same match status throughout the group.
+ if preserves_unmatched_probe
+ && !on
+ .iter()
+ .all(|(probe_key, _)| probe_exprs.contains(probe_key))
+ {
+ continue;
+ }
+ if !ordering_covers_unique_build_key(build, &on, &probe_exprs) {
+ continue;
+ }
+ valid_orderings.push(ordering.clone());
+ }
+ let mut probe_orderings = OrderingEquivalenceClass::new(valid_orderings);
+ if probe_orderings.is_empty() {
+ return Ok(probe_orderings);
+ }
+ let mut build_orderings = build.oeq_class().clone();
+ match probe_side {
+ JoinSide::Left =>
build_orderings.add_offset(probe.schema.fields().len() as _)?,
+ JoinSide::Right =>
probe_orderings.add_offset(build.schema.fields().len() as _)?,
+ JoinSide::None => unreachable!(),
+ }
+ Ok(probe_orderings.join_suffix(&build_orderings))
+}
+
+/// Check whether the probe ordering determines a unique build key.
+/// Join keys and probe ordering expressions must already be normalized.
+fn ordering_covers_unique_build_key(
+ build: &EquivalenceProperties,
+ on: &[(PhysicalExprRef, PhysicalExprRef)],
+ probe_exprs: &[PhysicalExprRef],
+) -> bool {
+ build.constraints().iter().any(|constraint| {
+ let (Constraint::PrimaryKey(indices) | Constraint::Unique(indices)) =
constraint;
+ !indices.is_empty()
+ && indices.iter().all(|&index| {
Review Comment:
[P2] Correct projected constraints before restoring ordering
Dropping or reordering a build-side primary key can incorrectly transfer its
uniqueness constraint to a duplicate join key. This guard then restores unsafe
ordering: LIMIT 2 returns [10,20] instead of [10,10]. The updated head fails 16
projection cases that the previous revision passed. The constraint-projection
defect predates this PR; the new exception reintroduces the original
wrong-result behavior. Correct or discard these projected constraints before
trusting them.
##########
datafusion/physical-expr/src/equivalence/properties/joins.rs:
##########
@@ -107,6 +91,96 @@ pub fn join_equivalence_properties(
Ok(result)
}
+/// Append build orderings only when equal probe ordering values identify at
most
+/// one build row. The suffix is then constant within each probe ordering
group,
+/// even if probe rows repeat. For an outer join preserving the probe side, all
+/// join keys must be fixed within the group, and the caller must rule out
filters,
+/// so the group cannot mix matched build rows with NULL-extended rows.
+fn unique_build_join_orderings(
+ probe: &EquivalenceProperties,
+ build: &EquivalenceProperties,
+ on: &[(PhysicalExprRef, PhysicalExprRef)],
+ probe_side: JoinSide,
+ preserves_unmatched_probe: bool,
+) -> Result<OrderingEquivalenceClass> {
+ if build.constraints().is_empty() || build.oeq_class().is_empty() {
+ return Ok(OrderingEquivalenceClass::default());
+ }
+ let on = on
+ .iter()
+ .map(|(left, right)| {
+ let (probe_key, build_key) = match probe_side {
+ JoinSide::Left => (left, right),
+ JoinSide::Right => (right, left),
+ JoinSide::None => unreachable!(),
+ };
+ (
+ probe.eq_group().normalize_expr(Arc::clone(probe_key)),
+ build.eq_group().normalize_expr(Arc::clone(build_key)),
+ )
+ })
+ .collect::<Vec<_>>();
+ let mut valid_orderings = Vec::new();
+ for ordering in probe.oeq_class().iter() {
+ let probe_exprs = ordering
+ .iter()
+ .map(|sort|
probe.eq_group().normalize_expr(Arc::clone(&sort.expr)))
+ .collect::<Vec<_>>();
+
+ // Outer joins must have the same match status throughout the group.
+ if preserves_unmatched_probe
+ && !on
+ .iter()
+ .all(|(probe_key, _)| probe_exprs.contains(probe_key))
+ {
+ continue;
+ }
+ if !ordering_covers_unique_build_key(build, &on, &probe_exprs) {
+ continue;
+ }
+ valid_orderings.push(ordering.clone());
+ }
+ let mut probe_orderings = OrderingEquivalenceClass::new(valid_orderings);
+ if probe_orderings.is_empty() {
+ return Ok(probe_orderings);
+ }
+ let mut build_orderings = build.oeq_class().clone();
+ match probe_side {
+ JoinSide::Left =>
build_orderings.add_offset(probe.schema.fields().len() as _)?,
+ JoinSide::Right =>
probe_orderings.add_offset(build.schema.fields().len() as _)?,
+ JoinSide::None => unreachable!(),
+ }
+ Ok(probe_orderings.join_suffix(&build_orderings))
+}
+
+/// Check whether the probe ordering determines a unique build key.
+/// Join keys and probe ordering expressions must already be normalized.
+fn ordering_covers_unique_build_key(
+ build: &EquivalenceProperties,
+ on: &[(PhysicalExprRef, PhysicalExprRef)],
+ probe_exprs: &[PhysicalExprRef],
+) -> bool {
+ build.constraints().iter().any(|constraint| {
+ let (Constraint::PrimaryKey(indices) | Constraint::Unique(indices)) =
constraint;
+ !indices.is_empty()
+ && indices.iter().all(|&index| {
+ let Some(field) = build.schema.fields().get(index) else {
+ return false;
+ };
+ // UNIQUE can contain repeated NULLs. Without null-equality
+ // information, only non-null UNIQUE columns prove uniqueness.
+ if matches!(constraint, Constraint::Unique(_)) &&
field.is_nullable() {
+ return false;
+ }
+ let column: PhysicalExprRef =
Arc::new(Column::new(field.name(), index));
+ let column = build.eq_group().normalize_expr(column);
+ on.iter().any(|(probe_key, build_key)| {
+ build_key.eq(&column) && probe_exprs.contains(probe_key)
+ })
Review Comment:
[P2] Recognize computed keys fixed by the ordering prefix
Exact expression membership misses ON build.k = probe.k + 1 when the probe
is ordered by k. Equal probe.k values determine the same primary-key build row,
making its suffix safe. Nevertheless, ORDER BY probe.k, build.k, build.v LIMIT
1 gains PartialSortExec and stalls on an unfinished group; the base returns
immediately. Check whether fixing the ordering expressions also fixes the
computed join key.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
