This is an automated email from the ASF dual-hosted git repository. hvanhovell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 229807efd06 [SPARK-42464][CONNECT] Fix ProtoToPlanTestSuite for Scala 2.13 229807efd06 is described below commit 229807efd06562fc07fce9a257062d8a66068761 Author: Herman van Hovell <her...@databricks.com> AuthorDate: Thu Feb 16 14:47:05 2023 -0400 [SPARK-42464][CONNECT] Fix ProtoToPlanTestSuite for Scala 2.13 ### What changes were proposed in this pull request? The `ProtoToPlanTestSuite` were broken for Scala 2.13 . This was caused by the following two problems: - Explain output between 2.12 and 2.13 is not stable because we render collection implementations as well. For this I changed the rendering of the offending classes to be version agnostic. - UDF code had deserialization issues. This was always the risk. I have removed those tests, we will work on improving UDF coverage in a follow-up. ### Why are the changes needed? We want to test Scala 2.13 as well. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Test only change. Closes #40055 from hvanhovell/SPARK-42464. Authored-by: Herman van Hovell <her...@databricks.com> Signed-off-by: Herman van Hovell <her...@databricks.com> --- .../apache/spark/sql/PlanGenerationTestSuite.scala | 21 ----- .../explain-results/column_as_multi.explain | 2 +- .../column_star_with_target.explain | 2 +- .../explain-results/function_udf.explain | 2 - .../explain-results/function_udf_2.12.explain | 2 - .../explain-results/function_udf_2.13.explain | 2 - .../join_inner_using_multiple_col_array.explain | 2 +- .../join_inner_using_multiple_col_seq.explain | 2 +- .../join_inner_using_single_col.explain | 2 +- .../join_using_multiple_col_array.explain | 2 +- .../join_using_multiple_col_seq.explain | 2 +- .../explain-results/join_using_single_col.explain | 2 +- .../query-tests/queries/function_udf.json | 96 --------------------- .../query-tests/queries/function_udf.proto.bin | Bin 11257 -> 0 bytes .../query-tests/queries/function_udf_2.12.json | 96 --------------------- .../queries/function_udf_2.12.proto.bin | Bin 11257 -> 0 bytes .../query-tests/queries/function_udf_2.13.json | 96 --------------------- .../queries/function_udf_2.13.proto.bin | Bin 12092 -> 0 bytes .../sql/connect/ProtoToParsedPlanTestSuite.scala | 5 -- .../spark/sql/catalyst/analysis/unresolved.scala | 4 +- .../spark/sql/catalyst/plans/joinTypes.scala | 1 + 21 files changed, 11 insertions(+), 330 deletions(-) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala index f8bae27cc0e..10aa4b658da 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala @@ -20,7 +20,6 @@ import java.nio.file.{Files, Path} import scala.collection.mutable import scala.util.{Failure, Success, Try} -import scala.util.Properties.versionNumberString import com.google.protobuf.util.JsonFormat import io.grpc.inprocess.InProcessChannelBuilder @@ -58,8 +57,6 @@ class PlanGenerationTestSuite extends ConnectFunSuite with BeforeAndAfterAll wit // Borrowed from SparkFunSuite private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1" - private val scala = versionNumberString.substring(0, versionNumberString.indexOf(".", 2)) - // Borrowed from SparkFunSuite private def getWorkspaceFilePath(first: String, more: String*): Path = { if (!(sys.props.contains("spark.test.home") || sys.env.contains("SPARK_HOME"))) { @@ -787,24 +784,6 @@ class PlanGenerationTestSuite extends ConnectFunSuite with BeforeAndAfterAll wit select(fn.max(Column("id"))) } - test("function udf " + scala) { - // This test might be a bit tricky if different JVM - // versions are used to generate the golden files. - val functions = Seq( - fn.udf(TestUDFs.udf0) - .asNonNullable() - .asNondeterministic(), - fn.udf(TestUDFs.udf1).withName("foo"), - fn.udf(TestUDFs.udf2).withName("f3"), - fn.udf(TestUDFs.udf3).withName("bar"), - fn.udf(TestUDFs.udf4).withName("f_four")) - val id = fn.col("id") - val columns = functions.zipWithIndex.map { case (udf, i) => - udf(Seq.fill(i)(id): _*) - } - select(columns: _*) - } - test("function lit") { select( fn.lit(fn.col("id")), diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/column_as_multi.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/column_as_multi.explain index 0afa06a3c1b..34d098320fa 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/column_as_multi.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/column_as_multi.explain @@ -1,2 +1,2 @@ -'Project ['d AS Buffer(v1, v2, v3)] +'Project ['d AS (v1, v2, v3)] +- LocalRelation <empty>, [none#0L, none#1, none#2, none#3, none#4, none#5, none#6] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/column_star_with_target.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/column_star_with_target.explain index 116393cfb02..f3366709711 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/column_star_with_target.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/column_star_with_target.explain @@ -1,2 +1,2 @@ -'Project [ArrayBuffer(str).*] +'Project [str.*] +- LocalRelation <empty>, [none#0L, none#1, none#2, none#3, none#4, none#5, none#6] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain deleted file mode 100644 index cdcf089bf7a..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain +++ /dev/null @@ -1,2 +0,0 @@ -'Project [unresolvedalias((), None), unresolvedalias(foo('id), None), unresolvedalias(f3('id, 'id), None), unresolvedalias(bar('id, 'id, 'id), None), unresolvedalias(f_four('id, 'id, 'id, 'id), None)] -+- LocalRelation <empty>, [none#0L, none#1, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain deleted file mode 100644 index cdcf089bf7a..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain +++ /dev/null @@ -1,2 +0,0 @@ -'Project [unresolvedalias((), None), unresolvedalias(foo('id), None), unresolvedalias(f3('id, 'id), None), unresolvedalias(bar('id, 'id, 'id), None), unresolvedalias(f_four('id, 'id, 'id, 'id), None)] -+- LocalRelation <empty>, [none#0L, none#1, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain deleted file mode 100644 index cdcf089bf7a..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain +++ /dev/null @@ -1,2 +0,0 @@ -'Project [unresolvedalias((), None), unresolvedalias(foo('id), None), unresolvedalias(f3('id, 'id), None), unresolvedalias(bar('id, 'id, 'id), None), unresolvedalias(f_four('id, 'id, 'id, 'id), None)] -+- LocalRelation <empty>, [none#0L, none#1, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_array.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_array.explain index 715fa26ba2e..8df79323f39 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_array.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_array.explain @@ -1,3 +1,3 @@ -'Join UsingJoin(Inner,Buffer(id, a)) +'Join UsingJoin(Inner, [id, a]) :- LocalRelation <empty>, [none#0L, none#1, none#2] +- LocalRelation <empty>, [none#0, none#1L, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_seq.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_seq.explain index 715fa26ba2e..8df79323f39 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_seq.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_seq.explain @@ -1,3 +1,3 @@ -'Join UsingJoin(Inner,Buffer(id, a)) +'Join UsingJoin(Inner, [id, a]) :- LocalRelation <empty>, [none#0L, none#1, none#2] +- LocalRelation <empty>, [none#0, none#1L, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_single_col.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_single_col.explain index bfcdcffbff9..8f83aaf56c1 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_single_col.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_single_col.explain @@ -1,3 +1,3 @@ -'Join UsingJoin(Inner,Buffer(id)) +'Join UsingJoin(Inner, [id]) :- LocalRelation <empty>, [none#0L, none#1, none#2] +- LocalRelation <empty>, [none#0, none#1L, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_array.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_array.explain index 92056649124..4b7bfe92868 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_array.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_array.explain @@ -1,3 +1,3 @@ -'Join UsingJoin(FullOuter,Buffer(id, a)) +'Join UsingJoin(FullOuter, [id, a]) :- LocalRelation <empty>, [none#0L, none#1, none#2] +- LocalRelation <empty>, [none#0, none#1L, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_seq.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_seq.explain index 577f544646e..7fb3e44a238 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_seq.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_seq.explain @@ -1,3 +1,3 @@ -'Join UsingJoin(RightOuter,Buffer(id, a)) +'Join UsingJoin(RightOuter, [id, a]) :- LocalRelation <empty>, [none#0L, none#1, none#2] +- LocalRelation <empty>, [none#0, none#1L, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_single_col.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_single_col.explain index 224790686d5..a5f9651e8be 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_single_col.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_single_col.explain @@ -1,3 +1,3 @@ -'Join UsingJoin(LeftSemi,Buffer(id)) +'Join UsingJoin(LeftSemi, [id]) :- LocalRelation <empty>, [none#0L, none#1, none#2] +- LocalRelation <empty>, [none#0, none#1L, none#2] diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf.json deleted file mode 100644 index 76738354e15..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "project": { - "input": { - "localRelation": { - "schema": "struct\u003cid:bigint,a:int,b:double\u003e" - } - }, - "expressions": [{ - "commonInlineUserDefinedFunction": { - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQxBwIoVHJpGRUCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "foo", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQyWQlE7Ce2cPkCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "f3", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQzyD4NN4Grh74CAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "bar", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQ009cpyjjQtFMCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "f_four", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQ1hQsS9jxAO/gCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - "nullable": true - } - } - }] - } -} \ No newline at end of file diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin deleted file mode 100644 index aa1971d3112..00000000000 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin and /dev/null differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json deleted file mode 100644 index 76738354e15..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "project": { - "input": { - "localRelation": { - "schema": "struct\u003cid:bigint,a:int,b:double\u003e" - } - }, - "expressions": [{ - "commonInlineUserDefinedFunction": { - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQxBwIoVHJpGRUCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "foo", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQyWQlE7Ce2cPkCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "f3", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQzyD4NN4Grh74CAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "bar", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQ009cpyjjQtFMCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "f_four", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQ1hQsS9jxAO/gCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...] - "nullable": true - } - } - }] - } -} \ No newline at end of file diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin deleted file mode 100644 index aa1971d3112..00000000000 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin and /dev/null differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json deleted file mode 100644 index 23aeb078543..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "project": { - "input": { - "localRelation": { - "schema": "struct\u003cid:bigint,a:int,b:double\u003e" - } - }, - "expressions": [{ - "commonInlineUserDefinedFunction": { - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kMQjucFe5xZ3dAgAAeHBzcgAyc2NhbGEuY29sbGVjdGlvbi5nZW5lcmljLkRlZmF1bHRTZXJpYWxpemF0aW9uUHJveHkAAAAAAAAAAwMAAUwAB2ZhY3Rvcn [...] - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "foo", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kMmifcExfceVGAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "f3", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kM6Rv/pH6bPiLAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "bar", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kNAv76myB5zZ4AgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...] - "nullable": true - } - } - }, { - "commonInlineUserDefinedFunction": { - "functionName": "f_four", - "deterministic": true, - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }, { - "unresolvedAttribute": { - "unparsedIdentifier": "id" - } - }], - "scalarScalaUdf": { - "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kNbf+MdXT5ejEAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...] - "nullable": true - } - } - }] - } -} \ No newline at end of file diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin deleted file mode 100644 index d7b3c84579c..00000000000 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin and /dev/null differ diff --git a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala index 1626d9278ff..8f47b7b7843 100644 --- a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala +++ b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala @@ -21,7 +21,6 @@ import java.nio.file.{Files, FileVisitResult, Path, SimpleFileVisitor} import java.nio.file.attribute.BasicFileAttributes import scala.util.{Failure, Success, Try} -import scala.util.Properties.versionNumberString import org.apache.spark.SparkFunSuite import org.apache.spark.connect.proto @@ -75,10 +74,6 @@ class ProtoToParsedPlanTestSuite extends SparkFunSuite with SharedSparkSession { private def createTest(file: Path): Unit = { val relativePath = inputFilePath.relativize(file) val fileName = relativePath.getFileName.toString - if ((versionNumberString.startsWith("2.12") && fileName.endsWith("2.13.proto.bin")) || - (versionNumberString.startsWith("2.13") && fileName.endsWith("2.12.proto.bin"))) { - return - } if (!fileName.endsWith(".proto.bin")) { logError(s"Skipping $fileName") return diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index 5e20f12747b..04bc513aa99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -459,7 +459,7 @@ case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevalu } } - override def toString: String = target.map(_ + ".").getOrElse("") + "*" + override def toString: String = target.map(_.mkString("", ".", ".")).getOrElse("") + "*" } /** @@ -516,7 +516,7 @@ case class MultiAlias(child: Expression, names: Seq[String]) override lazy val resolved = false - override def toString: String = s"$child AS $names" + override def toString: String = s"$child AS ${names.mkString("(", ", ", ")")}" override protected def withNewChildInternal(newChild: Expression): MultiAlias = copy(child = newChild) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala index eeec3cd765a..0c5f620d722 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala @@ -105,6 +105,7 @@ case class UsingJoin(tpe: JoinType, usingColumns: Seq[String]) extends JoinType require(Seq(Inner, LeftOuter, LeftSemi, RightOuter, FullOuter, LeftAnti, Cross).contains(tpe), "Unsupported using join type " + tpe) override def sql: String = "USING " + tpe.sql + override def toString: String = s"UsingJoin($tpe, ${usingColumns.mkString("[", ", ", "]")})" } object LeftExistence { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org