Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/19813#discussion_r154953139
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
---
@@ -236,4 +237,22 @@ class WholeStageCodegenSuite extends QueryTest with
SharedSQLContext {
}
}
}
+
+ test("SPARK-22551: Fix 64kb limit for deeply nested expressions under
wholestage codegen") {
+ import testImplicits._
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+ val df = Seq(("abc", 1)).toDF("key", "int")
+ df.write.parquet(path)
+
+ var strExpr: Expression = col("key").expr
+ for (_ <- 1 to 150) {
+ strExpr = Decode(Encode(strExpr, Literal("utf-8")),
Literal("utf-8"))
+ }
+ val expressions = Seq(If(EqualTo(strExpr, strExpr), strExpr,
strExpr))
+
+ val df2 =
spark.read.parquet(path).select(expressions.map(Column(_)): _*)
--- End diff --
Yes.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
