Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/3778#discussion_r22274369
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -293,6 +295,380 @@ object OptimizeIn extends Rule[LogicalPlan] {
}
}
+object ConditionSimplification extends Rule[LogicalPlan] {
+ import BinaryComparison.LiteralComparison
+
+ def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+ case q: LogicalPlan => q transformExpressionsDown {
+ case origin: CombinePredicate =>
+ origin.toOptimized
+ }
+ }
+
+ type SplitFragments = Map[Expression, Option[Expression]]
+
+ implicit class CombinePredicateExtension(source: CombinePredicate) {
+ def find(goal: Expression): Boolean = {
+ def delegate(child: Expression): Boolean = (child, goal) match {
+ case (combine: CombinePredicate, _) =>
+ isSameCombinePredicate(source, combine) && combine.find(goal)
+
+ // if left child is a literal
+ // LiteralComparison's unapply method change the literal and
attribute position
+ case (LiteralComparison(childComparison),
LiteralComparison(goalComparison)) =>
+ isSame(childComparison, goalComparison)
+
+ case other =>
+ isSame(child, goal)
+ }
+
+ // using method to avoid right side compute if left side is true
+ val leftResult = () => delegate(source.left)
+ val rightResult = () => delegate(source.right)
+ leftResult() || rightResult()
+ }
+
+ @inline
+ def isOrPredicate: Boolean = {
+ source.isInstanceOf[Or]
+ }
+
+ // create a new combine predicate that has the same combine operator
as this
+ @inline
+ def build(left: Expression, right: Expression): CombinePredicate = {
+ CombinePredicate(left, right, isOrPredicate)
+ }
+
+ // swap left child and right child
+ @inline
+ def swap: CombinePredicate = {
+ source.build(source.right, source.left)
+ }
+
+ def toOptimized: Expression = source match {
+ // one CombinePredicate, left equals right , drop right, keep left
+ // examples: a && a => a, a || a => a
+ case CombinePredicate(left, right) if left.fastEquals(right) =>
+ left
+
+ // one CombinePredicate and left and right are both binary comparison
+ // examples: a < 2 && a > 2 => false
+ case origin @ CombinePredicate(LiteralComparison(left),
LiteralComparison(right)) =>
+ // left or right maybe change its child position, so rebuild one
+ val changed = origin.build(left, right)
+ val optimized = changed.mergeComparison
+ if (isSame(changed, optimized)) {
+ origin
+ } else {
+ optimized
+ }
+
+ case origin @ CombinePredicate(left @ CombinePredicate(ll, lr),
right)
+ if isNotCombinePredicate(ll, lr, right) =>
+ val leftOptimized = left.toOptimized
+ if (isSame(left, leftOptimized)) {
+ if (isSame(ll, right) || isSame(lr, right)) {
+ if (isSameCombinePredicate(origin, left)) leftOptimized else
right
+ } else {
+ val llRight = origin.build(ll, right)
+ val lrRight = origin.build(lr, right)
+ val llRightOptimized = llRight.toOptimized
+ val lrRightOptimized = lrRight.toOptimized
+ if (isSame(llRight, llRightOptimized) && isSame(lrRight,
lrRightOptimized)) {
+ origin
+ } else if ((isNotCombinePredicate(llRightOptimized,
lrRightOptimized))
+ || isSameCombinePredicate(origin, left)) {
+ left.build(llRightOptimized, lrRightOptimized).toOptimized
+ } else if (llRightOptimized.isLiteral ||
lrRightOptimized.isLiteral) {
+ left.build(llRightOptimized, lrRightOptimized)
+ } else {
+ origin
+ }
+ }
+ } else if (isNotCombinePredicate(leftOptimized)) {
+ origin.build(leftOptimized, right).toOptimized
+ } else {
+ origin
+ }
+
+ case origin @ CombinePredicate(left, right @ CombinePredicate(left2,
right2))
+ if isNotCombinePredicate(left, left2, right2) =>
+ val changed = origin.swap
+ val optimized = changed.toOptimized
+ if (isSame(changed, optimized)) {
+ origin
+ } else {
+ optimized
+ }
+
+ // do optimize like : (a || b || c) && a => a, here a, b , c is a
condition
+ case origin @ CombinePredicate(left: CombinePredicate, right) =>
+ val leftOptimized = left.toOptimized
+ val rightOptimized = if (right.isCombinePredicate) {
+ right.toCombinePredicate.toOptimized
+ } else {
+ right
+ }
+ if (!isSame(left, leftOptimized) || !isSame(right,
rightOptimized)) {
+ origin.build(leftOptimized, rightOptimized).toOptimized
+ } else {
+ val rightFragments = right.split.iterator
+ val leftFragments = left.split
+ while (rightFragments.hasNext) {
+ val (rightUnit, rightRemainFragment) = rightFragments.next()
+ val leftRemainFragment = leftFragments.getOrElse(rightUnit,
None)
+ if (leftRemainFragment.isDefined) {
+ right match {
+ case rightCombine: CombinePredicate =>
+ if (isSameCombinePredicate(origin, left,
rightCombine)) {
+ return leftRemainFragment
+ .map(leftRemain => origin.build(leftRemain,
right).toOptimized)
+ .getOrElse(right)
+ } else if (isSameCombinePredicate(origin,
rightCombine)) {
+ return right
+ } else if (isSameCombinePredicate(origin, left)) {
+ return left
+ } else {
+ (leftRemainFragment, rightRemainFragment) match {
+ case (Some(leftRemain), Some(rightRemain)) =>
+ val remains = origin.build(leftRemain,
rightRemain)
+ return left.build(rightUnit, remains)
+
+ case (Some(leftRemain), None) =>
+ return right
+
+ case (_, _) =>
+ return left
+ }
+ }
+
+ case _ =>
+ if (isSameCombinePredicate(origin, left)) {
+ return left
+ } else {
+ return right
+ }
+ }
+ }
+
+ }
+ // here can add more complicated case process code
+ origin
+
+ }
+
+ case origin @ CombinePredicate(left, right @ CombinePredicate(rl,
rr))
+ if isNotCombinePredicate(left) =>
+ val changed = origin.swap
+ val optimized = changed.toOptimized
+ if (isSame(changed, optimized)) {
+ origin
+ } else {
+ optimized
+ }
+
+ case other =>
+ other
+ }
+ // merge to literal comparison(contains literal in binary comparison)
+ // here assume two children both are `BinaryComparison`
+ // TODO : find a way to simplify the code O_o
+ def mergeComparison: Expression = {
+ val left = source.left.toBinaryComparison
+ val right = source.right.toBinaryComparison
+ if (!isSame(left.left, right.left)) {
+ source
+ } else {
+ (left, right) match {
+ case (BinaryComparison(attr, Literal(vLeft, tLeft @
NativeType())),
+ BinaryComparison(_, Literal(vRight, tRight @ NativeType()))) =>
+ val result = compare(vLeft, vRight)
+ result.filter(_ > 0).map(c => {
+ if (((left.isLess || left.isLessEquals)
+ && (right.isLess || right.isLessEquals ||
right.isEquals))) {
--- End diff --
Usually similar deeply nested `if` statements can be refactored into
carefully organized pattern matches. Using pattern matching also frees you from
helper methods like `.isLess` and `isLessEquals`, thus `ExpressionCookies`
won't be necessary.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
