Repository: spark Updated Branches: refs/heads/master f876d3fa8 -> 0ca16f6e1
Revert "[SPARK-24402][SQL] Optimize `In` expression when only one element in the collection or collection is empty" This reverts commit 0f0d1865f581a9158d73505471953656b173beba. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0ca16f6e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0ca16f6e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0ca16f6e Branch: refs/heads/master Commit: 0ca16f6e143768f0c96b5310c1f81b3b51dcbbc8 Parents: f876d3f Author: hyukjinkwon <[email protected]> Authored: Tue Jul 17 11:30:53 2018 +0800 Committer: hyukjinkwon <[email protected]> Committed: Tue Jul 17 11:30:53 2018 +0800 ---------------------------------------------------------------------- .../sql/catalyst/optimizer/expressions.scala | 13 +++----- .../catalyst/optimizer/OptimizeInSuite.scala | 32 -------------------- 2 files changed, 4 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/0ca16f6e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala index f78a0ff..1d363b8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala @@ -218,20 +218,15 @@ object ReorderAssociativeOperator extends Rule[LogicalPlan] { object OptimizeIn extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transform { case q: LogicalPlan => q transformExpressionsDown { - case In(v, list) if list.isEmpty => - // When v is not nullable, the following expression will be optimized - // to FalseLiteral which is tested in OptimizeInSuite.scala - If(IsNotNull(v), FalseLiteral, Literal(null, BooleanType)) + case In(v, list) if list.isEmpty && !v.nullable => FalseLiteral case expr @ In(v, list) if expr.inSetConvertible => val newList = ExpressionSet(list).toSeq - if (newList.length == 1 && !newList.isInstanceOf[ListQuery]) { - EqualTo(v, newList.head) - } else if (newList.length > SQLConf.get.optimizerInSetConversionThreshold) { + if (newList.size > SQLConf.get.optimizerInSetConversionThreshold) { val hSet = newList.map(e => e.eval(EmptyRow)) InSet(v, HashSet() ++ hSet) - } else if (newList.length < list.length) { + } else if (newList.size < list.size) { expr.copy(list = newList) - } else { // newList.length == list.length && newList.length > 1 + } else { // newList.length == list.length expr } } http://git-wip-us.apache.org/repos/asf/spark/blob/0ca16f6e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala index 86522a6..478118e 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala @@ -176,21 +176,6 @@ class OptimizeInSuite extends PlanTest { } } - test("OptimizedIn test: one element in list gets transformed to EqualTo.") { - val originalQuery = - testRelation - .where(In(UnresolvedAttribute("a"), Seq(Literal(1)))) - .analyze - - val optimized = Optimize.execute(originalQuery) - val correctAnswer = - testRelation - .where(EqualTo(UnresolvedAttribute("a"), Literal(1))) - .analyze - - comparePlans(optimized, correctAnswer) - } - test("OptimizedIn test: In empty list gets transformed to FalseLiteral " + "when value is not nullable") { val originalQuery = @@ -206,21 +191,4 @@ class OptimizeInSuite extends PlanTest { comparePlans(optimized, correctAnswer) } - - test("OptimizedIn test: In empty list gets transformed to `If` expression " + - "when value is nullable") { - val originalQuery = - testRelation - .where(In(UnresolvedAttribute("a"), Nil)) - .analyze - - val optimized = Optimize.execute(originalQuery) - val correctAnswer = - testRelation - .where(If(IsNotNull(UnresolvedAttribute("a")), - Literal(false), Literal.create(null, BooleanType))) - .analyze - - comparePlans(optimized, correctAnswer) - } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
