cloud-fan commented on a change in pull request #33281:
URL: https://github.com/apache/spark/pull/33281#discussion_r746630300
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EquivalentExpressions.scala
##########
@@ -38,19 +41,41 @@ class EquivalentExpressions {
* Returns true if there was already a matching expression.
*/
def addExpr(expr: Expression): Boolean = {
- addExprToMap(expr, equivalenceMap)
+ updateExprInMap(expr, equivalenceMap)
}
- private def addExprToMap(
- expr: Expression, map: mutable.HashMap[ExpressionEquals,
ExpressionStats]): Boolean = {
+ /**
+ * Adds or removes an expression to/from the map and updates `useCount`.
+ * Returns true
+ * - if there was a matching expression in the map before add or
+ * - if there remained a matching expression in the map after remove
(`useCount` remained > 0)
+ * to indicate there is no need to recurse in `updateExprTree`.
+ */
+ private def updateExprInMap(
+ expr: Expression,
+ map: mutable.HashMap[ExpressionEquals, ExpressionStats],
+ useCount: Int = 1): Boolean = {
if (expr.deterministic) {
val wrapper = ExpressionEquals(expr)
map.get(wrapper) match {
case Some(stats) =>
- stats.useCount += 1
- true
+ stats.useCount += useCount
+ if (stats.useCount > 0) {
+ true
+ } else if (stats.useCount == 0) {
+ map -= wrapper
+ false
+ } else {
+ // Should not happen
Review comment:
If this should not happen, I think throwing `IllegalStateException` is
better as it's a bug. `QueryExecutionErrors` is for user-facing errors.
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EquivalentExpressions.scala
##########
@@ -38,19 +41,41 @@ class EquivalentExpressions {
* Returns true if there was already a matching expression.
*/
def addExpr(expr: Expression): Boolean = {
- addExprToMap(expr, equivalenceMap)
+ updateExprInMap(expr, equivalenceMap)
}
- private def addExprToMap(
- expr: Expression, map: mutable.HashMap[ExpressionEquals,
ExpressionStats]): Boolean = {
+ /**
+ * Adds or removes an expression to/from the map and updates `useCount`.
+ * Returns true
+ * - if there was a matching expression in the map before add or
+ * - if there remained a matching expression in the map after remove
(`useCount` remained > 0)
+ * to indicate there is no need to recurse in `updateExprTree`.
+ */
+ private def updateExprInMap(
+ expr: Expression,
+ map: mutable.HashMap[ExpressionEquals, ExpressionStats],
+ useCount: Int = 1): Boolean = {
if (expr.deterministic) {
val wrapper = ExpressionEquals(expr)
map.get(wrapper) match {
case Some(stats) =>
- stats.useCount += 1
- true
+ stats.useCount += useCount
+ if (stats.useCount > 0) {
+ true
+ } else if (stats.useCount == 0) {
+ map -= wrapper
+ false
+ } else {
+ // Should not happen
+ throw QueryExecutionErrors.updateEquivalentExpressionsError(expr,
map, useCount)
+ }
case _ =>
- map.put(wrapper, ExpressionStats(expr)())
+ if (useCount > 0) {
+ map.put(wrapper, ExpressionStats(expr)(useCount))
+ } else {
+ // Should not happen
+ throw QueryExecutionErrors.updateEquivalentExpressionsError(expr,
map, useCount)
Review comment:
ditto
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EquivalentExpressions.scala
##########
@@ -59,45 +84,43 @@ class EquivalentExpressions {
}
/**
- * Adds only expressions which are common in each of given expressions, in a
recursive way.
- * For example, given two expressions `(a + (b + (c + 1)))` and `(d + (e +
(c + 1)))`,
- * the common expression `(c + 1)` will be added into `equivalenceMap`.
+ * Adds or removes only expressions which are common in each of given
expressions, in a recursive
Review comment:
It may remove expressions from the given `map`? I think it only adds.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
