This is an automated email from the ASF dual-hosted git repository. yumwang pushed a commit to branch branch-3.5 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push: new 41e7234b848 [SPARK-44834][PYTHON][SQL][TESTS][FOLLOW-UP] Update the analyzer results of the udtf tests 41e7234b848 is described below commit 41e7234b848908afa12a8cc4a319b214a461c12d Author: allisonwang-db <allison.w...@databricks.com> AuthorDate: Fri Aug 18 16:31:42 2023 +0800 [SPARK-44834][PYTHON][SQL][TESTS][FOLLOW-UP] Update the analyzer results of the udtf tests ### What changes were proposed in this pull request? This is a follow up for https://github.com/apache/spark/pull/42517. We need to re-generate the analyzer results for udtf tests after https://github.com/apache/spark/pull/42519 is merged. Also updated PythonUDTFSuite after https://github.com/apache/spark/pull/42520 is merged. ### Why are the changes needed? To fix test failures ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Test only change Closes #42543 from allisonwang-db/spark-44834-fix. Authored-by: allisonwang-db <allison.w...@databricks.com> Signed-off-by: Yuming Wang <yumw...@ebay.com> (cherry picked from commit bb41cd889efdd0602385e70b4c8f1c93740db332) Signed-off-by: Yuming Wang <yumw...@ebay.com> --- .../sql-tests/analyzer-results/udtf/udtf.sql.out | 51 ++++------------------ .../sql/execution/python/PythonUDTFSuite.scala | 17 +------- 2 files changed, 10 insertions(+), 58 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/udtf/udtf.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/udtf/udtf.sql.out index acf96794378..b46a1f230a8 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/udtf/udtf.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/udtf/udtf.sql.out @@ -10,84 +10,49 @@ CreateViewCommand `t1`, VALUES (0, 1), (1, 2) t(c1, c2), false, true, LocalTempV -- !query SELECT * FROM udtf(1, 2) -- !query analysis -Project [x#x, y#x] -+- Generate TestUDTF(1, 2)#x, false, [x#x, y#x] - +- OneRowRelation +[Analyzer test output redacted due to nondeterminism] -- !query SELECT * FROM udtf(-1, 0) -- !query analysis -Project [x#x, y#x] -+- Generate TestUDTF(-1, 0)#x, false, [x#x, y#x] - +- OneRowRelation +[Analyzer test output redacted due to nondeterminism] -- !query SELECT * FROM udtf(0, -1) -- !query analysis -Project [x#x, y#x] -+- Generate TestUDTF(0, -1)#x, false, [x#x, y#x] - +- OneRowRelation +[Analyzer test output redacted due to nondeterminism] -- !query SELECT * FROM udtf(0, 0) -- !query analysis -Project [x#x, y#x] -+- Generate TestUDTF(0, 0)#x, false, [x#x, y#x] - +- OneRowRelation +[Analyzer test output redacted due to nondeterminism] -- !query SELECT a, b FROM udtf(1, 2) t(a, b) -- !query analysis -Project [a#x, b#x] -+- SubqueryAlias t - +- Project [x#x AS a#x, y#x AS b#x] - +- Generate TestUDTF(1, 2)#x, false, [x#x, y#x] - +- OneRowRelation +[Analyzer test output redacted due to nondeterminism] -- !query SELECT * FROM t1, LATERAL udtf(c1, c2) -- !query analysis -Project [c1#x, c2#x, x#x, y#x] -+- LateralJoin lateral-subquery#x [c1#x && c2#x], Inner - : +- Generate TestUDTF(outer(c1#x), outer(c2#x))#x, false, [x#x, y#x] - : +- OneRowRelation - +- SubqueryAlias t1 - +- View (`t1`, [c1#x,c2#x]) - +- Project [cast(c1#x as int) AS c1#x, cast(c2#x as int) AS c2#x] - +- SubqueryAlias t - +- LocalRelation [c1#x, c2#x] +[Analyzer test output redacted due to nondeterminism] -- !query SELECT * FROM t1 LEFT JOIN LATERAL udtf(c1, c2) -- !query analysis -Project [c1#x, c2#x, x#x, y#x] -+- LateralJoin lateral-subquery#x [c1#x && c2#x], LeftOuter - : +- Generate TestUDTF(outer(c1#x), outer(c2#x))#x, false, [x#x, y#x] - : +- OneRowRelation - +- SubqueryAlias t1 - +- View (`t1`, [c1#x,c2#x]) - +- Project [cast(c1#x as int) AS c1#x, cast(c2#x as int) AS c2#x] - +- SubqueryAlias t - +- LocalRelation [c1#x, c2#x] +[Analyzer test output redacted due to nondeterminism] -- !query SELECT * FROM udtf(1, 2) t(c1, c2), LATERAL udtf(c1, c2) -- !query analysis -Project [c1#x, c2#x, x#x, y#x] -+- LateralJoin lateral-subquery#x [c1#x && c2#x], Inner - : +- Generate TestUDTF(outer(c1#x), outer(c2#x))#x, false, [x#x, y#x] - : +- OneRowRelation - +- SubqueryAlias t - +- Project [x#x AS c1#x, y#x AS c2#x] - +- Generate TestUDTF(1, 2)#x, false, [x#x, y#x] - +- OneRowRelation +[Analyzer test output redacted due to nondeterminism] -- !query diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDTFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDTFSuite.scala index 67d33c6ecdf..a9d56ce9bc9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDTFSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDTFSuite.scala @@ -39,19 +39,6 @@ class PythonUDTFSuite extends QueryTest with SharedSparkSession { | yield a, b, b - a |""".stripMargin - private val arrowPythonScript: String = - """ - |import pandas as pd - |class VectorizedUDTF: - | def eval(self, a: pd.Series, b: pd.Series): - | data = [ - | [a, b, a + b], - | [a, b, a - b], - | [a, b, b - a], - | ] - | yield pd.DataFrame(data) - |""".stripMargin - private val returnType: StructType = StructType.fromDDL("a int, b int, c int") private val pythonUDTF: UserDefinedPythonTableFunction = @@ -59,8 +46,8 @@ class PythonUDTFSuite extends QueryTest with SharedSparkSession { private val arrowPythonUDTF: UserDefinedPythonTableFunction = createUserDefinedPythonTableFunction( - "VectorizedUDTF", - arrowPythonScript, + "SimpleUDTF", + pythonScript, returnType, evalType = PythonEvalType.SQL_ARROW_TABLE_UDF) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org