This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new e75fabf Revert "[SPARK-35636][SQL] Lambda keys should not be
referenced outside of the lambda function"
e75fabf is described below
commit e75fabf3c6651fa299fa5dc6ca2539b4f61fa710
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Jun 4 16:32:09 2021 +0900
Revert "[SPARK-35636][SQL] Lambda keys should not be referenced outside of
the lambda function"
This reverts commit f10fbb2055ecf4163b5b3d236e69138dfb228e1a.
---
.../expressions/higherOrderFunctions.scala | 10 -------
.../optimizer/NestedColumnAliasingSuite.scala | 32 +---------------------
2 files changed, 1 insertion(+), 41 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index 0a6376f..a4e069d 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -84,9 +84,6 @@ case class NamedLambdaVariable(
AttributeReference(name, dataType, nullable, Metadata.empty)(exprId,
Seq.empty)
}
- // Check if this lambda variable is referenced outside the lambda function
it is bound to
- override def references: AttributeSet = AttributeSet(toAttribute)
-
override def eval(input: InternalRow): Any = value.get
override def toString: String = s"lambda $name#${exprId.id}$typeSuffix"
@@ -111,13 +108,6 @@ case class LambdaFunction(
override def dataType: DataType = function.dataType
override def nullable: Boolean = function.nullable
- // Check if lambda variables bound to this lambda function are referenced in
the wrong scope
- override def references: AttributeSet = if (resolved) {
- function.references -- AttributeSet(arguments.flatMap(_.references))
- } else {
- super.references
- }
-
lazy val bound: Boolean = arguments.forall(_.resolved)
override def eval(input: InternalRow): Any = function.eval(input)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
index 08e442d..c83ab37 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
@@ -23,10 +23,9 @@ import org.apache.spark.sql.catalyst.SchemaPruningTest
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.Cross
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.RuleExecutor
-import org.apache.spark.sql.types.{IntegerType, StringType, StructField,
StructType}
+import org.apache.spark.sql.types.{StringType, StructField, StructType}
class NestedColumnAliasingSuite extends SchemaPruningTest {
@@ -685,35 +684,6 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
).analyze
comparePlans(optimized2, expected2)
}
-
- test("SPARK-35636: do not push lambda key out of lambda function") {
- val rel = LocalRelation(
- 'kvs.map(StringType, new StructType().add("v1", IntegerType)),
'keys.array(StringType))
- val key = UnresolvedNamedLambdaVariable("key" :: Nil)
- val lambda = LambdaFunction('kvs.getItem(key).getField("v1"), key :: Nil)
- val query = rel
- .limit(5)
- .select('keys, 'kvs)
- .limit(5)
- .select(ArrayTransform('keys, lambda).as("a"))
- .analyze
- val optimized = Optimize.execute(query)
- comparePlans(optimized, query)
- }
-
- test("SPARK-35636: do not push down extract value in higher order " +
- "function that references both sides of a join") {
- val left = LocalRelation('kvs.map(StringType, new StructType().add("v1",
IntegerType)))
- val right = LocalRelation('keys.array(StringType))
- val key = UnresolvedNamedLambdaVariable("key" :: Nil)
- val lambda = LambdaFunction('kvs.getItem(key).getField("v1"), key :: Nil)
- val query = left
- .join(right, Cross, None)
- .select(ArrayTransform('keys, lambda).as("a"))
- .analyze
- val optimized = Optimize.execute(query)
- comparePlans(optimized, query)
- }
}
object NestedColumnAliasingSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
