Github user maryannxue commented on a diff in the pull request:
https://github.com/apache/spark/pull/21764#discussion_r202760924
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -175,6 +179,35 @@ abstract class Optimizer(sessionCatalog:
SessionCatalog)
* Override to provide additional rules for the operator optimization
batch.
*/
def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] = Nil
+
+ override def batches: Seq[Batch] = {
+ val excludedRules =
+
SQLConf.get.optimizerExcludedRules.toSeq.flatMap(_.split(",").map(_.trim).filter(!_.isEmpty))
+ val filteredOptimizationBatches = if (excludedRules.isEmpty) {
+ optimizationBatches
+ } else {
+ optimizationBatches.flatMap { batch =>
+ val filteredRules =
+ batch.rules.filter { rule =>
+ val exclude = excludedRules.contains(rule.ruleName)
+ if (exclude) {
+ logInfo(s"Optimization rule '${rule.ruleName}' is excluded
from the optimizer.")
+ }
+ !exclude
+ }
+ if (batch.rules == filteredRules) {
--- End diff --
It is to:
1) avoid unnecessary object creation if all rules have been preserved.
2) avoid empty batches if all rules in the batch have been removed.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
