peter-toth commented on code in PR #38035:
URL: https://github.com/apache/spark/pull/38035#discussion_r1105972190
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala:
##########
@@ -183,26 +186,95 @@ trait LeafNode extends LogicalPlan with
LeafLike[LogicalPlan] {
* A logical plan node with single child.
*/
trait UnaryNode extends LogicalPlan with UnaryLike[LogicalPlan] {
+ val constraintProjectionLimit =
conf.getConf(SQLConf.CONSTRAINT_PROJECTION_LIMIT)
+
/**
- * Generates all valid constraints including an set of aliased constraints
by replacing the
- * original constraint expressions with the corresponding alias
+ * Generates all valid constraints including a set of aliased constraints by
replacing the
+ * original constraint expressions with the corresponding alias.
+ * This method only returns constraints whose referenced attributes are
subset of `outputSet`.
*/
protected def getAllValidConstraints(projectList: Seq[NamedExpression]):
ExpressionSet = {
- var allConstraints = child.constraints
+ val newLiteralConstraints = mutable.ArrayBuffer.empty[EqualNullSafe]
+ val newConstraints = mutable.ArrayBuffer.empty[EqualNullSafe]
+ val aliasMap = mutable.Map[Expression, mutable.ArrayBuffer[Attribute]]()
projectList.foreach {
+ // These new literal constraints doesn't need any projection, not they
can project any other
+ // constraint
case a @ Alias(l: Literal, _) =>
- allConstraints += EqualNullSafe(a.toAttribute, l)
- case a @ Alias(e, _) if e.deterministic =>
- // For every alias in `projectList`, replace the reference in
constraints by its attribute.
- allConstraints ++= allConstraints.map(_ transform {
- case expr: Expression if expr.semanticEquals(e) =>
- a.toAttribute
- })
- allConstraints += EqualNullSafe(e, a.toAttribute)
- case _ => // Don't change.
+ newLiteralConstraints += EqualNullSafe(a.toAttribute, l)
+
+ // We need to add simple attributes to the alias as those attributes can
be aliased as well.
+ // Technically, we don't need to add attributes that are not otherwise
aliased to the map, but
+ // adding them does no harm.
+ case a: Attribute =>
+ aliasMap.getOrElseUpdate(a.canonicalized, mutable.ArrayBuffer.empty)
+= a
+
+ // If we have an alias in the projection then we need to:
+ // - add it to the alias map as it can project child's constraints
+ // - and add it to the new constraints and let it be projected or pruned
based on other
+ // aliases and attributes in the project list.
+ // E.g. `a + b <=> x` constraint can "survive" the projection if
+ // - `a + b` is aliased (like `a + b AS y` and so the projected
constraint is `y <=> x`)
+ // - or both `a` and `b` are aliased or included in the output set
+ case a @ Alias(child, _) if child.deterministic =>
+ val attr = a.toAttribute
+ aliasMap.getOrElseUpdate(child.canonicalized,
mutable.ArrayBuffer.empty) += attr
+ newConstraints += EqualNullSafe(child, attr)
+ case _ =>
+ }
+
+ def projectConstraint(expr: Expression) = {
+ // The current constraint projection doesn't do a full-blown projection
which means that when
Review Comment:
I wanted to keep the current beahaviour in this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
