This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 07816316c265 [SPARK-49439][SQL] Fix the pretty name of the
`FromProtobuf` & `ToProtobuf` expression
07816316c265 is described below
commit 07816316c2658bc4736f707e18a119e5cfc936c3
Author: panbingkun <[email protected]>
AuthorDate: Wed Aug 28 09:34:32 2024 +0200
[SPARK-49439][SQL] Fix the pretty name of the `FromProtobuf` & `ToProtobuf`
expression
### What changes were proposed in this pull request?
In the PR, I propose to override the `prettyName` method of the
`FromProtobuf` & `ToProtobuf` expression and set to `from_protobuf` &
`to_protobuf` by default as in `FunctionRegistry`:
https://github.com/apache/spark/blob/6d8235f3b2bbaa88b10c35d6eecddffa4d1b04a4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala#L877-L878
### Why are the changes needed?
To don't confuse users by non-existent function name, and print correct
name in errors.
### Does this PR introduce _any_ user-facing change?
Yes.
### How was this patch tested?
Update existed UT.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47901 from panbingkun/SPARK-49439.
Authored-by: panbingkun <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
.../apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala | 12 ++++++------
.../catalyst/expressions/toFromProtobufSqlFunctions.scala | 7 +++++++
2 files changed, 13 insertions(+), 6 deletions(-)
diff --git
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
index ec57ffa699d9..4ff432cf7a05 100644
---
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
+++
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
@@ -2095,7 +2095,7 @@ class ProtobufFunctionsSuite extends QueryTest with
SharedSparkSession with Prot
|""".stripMargin)),
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map(
- "sqlExpr" -> s"""\"toprotobuf(complex_struct, 42, $testFileDescFile,
map())\"""",
+ "sqlExpr" -> s"""\"to_protobuf(complex_struct, 42,
$testFileDescFile, map())\"""",
"msg" -> ("The second argument of the TO_PROTOBUF SQL function must
be a constant " +
"string representing the Protobuf message name"),
"hint" -> ""),
@@ -2113,7 +2113,7 @@ class ProtobufFunctionsSuite extends QueryTest with
SharedSparkSession with Prot
|""".stripMargin)),
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map(
- "sqlExpr" -> "\"toprotobuf(complex_struct, SimpleMessageJavaTypes,
42, map())\"",
+ "sqlExpr" -> "\"to_protobuf(complex_struct, SimpleMessageJavaTypes,
42, map())\"",
"msg" -> ("The third argument of the TO_PROTOBUF SQL function must
be a constant " +
"string representing the Protobuf descriptor file path"),
"hint" -> ""),
@@ -2133,7 +2133,7 @@ class ProtobufFunctionsSuite extends QueryTest with
SharedSparkSession with Prot
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map(
"sqlExpr" ->
- s"""\"toprotobuf(complex_struct, SimpleMessageJavaTypes,
$testFileDescFile, 42)\"""",
+ s"""\"to_protobuf(complex_struct, SimpleMessageJavaTypes,
$testFileDescFile, 42)\"""",
"msg" -> ("The fourth argument of the TO_PROTOBUF SQL function must
be a constant " +
"map of strings to strings containing the options to use for
converting the value " +
"to Protobuf format"),
@@ -2154,7 +2154,7 @@ class ProtobufFunctionsSuite extends QueryTest with
SharedSparkSession with Prot
|""".stripMargin)),
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map(
- "sqlExpr" -> s"""\"fromprotobuf(protobuf_data, 42,
$testFileDescFile, map())\"""",
+ "sqlExpr" -> s"""\"from_protobuf(protobuf_data, 42,
$testFileDescFile, map())\"""",
"msg" -> ("The second argument of the FROM_PROTOBUF SQL function
must be a constant " +
"string representing the Protobuf message name"),
"hint" -> ""),
@@ -2171,7 +2171,7 @@ class ProtobufFunctionsSuite extends QueryTest with
SharedSparkSession with Prot
|""".stripMargin)),
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map(
- "sqlExpr" -> "\"fromprotobuf(protobuf_data, SimpleMessageJavaTypes,
42, map())\"",
+ "sqlExpr" -> "\"from_protobuf(protobuf_data, SimpleMessageJavaTypes,
42, map())\"",
"msg" -> ("The third argument of the FROM_PROTOBUF SQL function must
be a constant " +
"string representing the Protobuf descriptor file path"),
"hint" -> ""),
@@ -2191,7 +2191,7 @@ class ProtobufFunctionsSuite extends QueryTest with
SharedSparkSession with Prot
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map(
"sqlExpr" ->
- s"""\"fromprotobuf(protobuf_data, SimpleMessageJavaTypes,
$testFileDescFile, 42)\"""",
+ s"""\"from_protobuf(protobuf_data, SimpleMessageJavaTypes,
$testFileDescFile, 42)\"""",
"msg" -> ("The fourth argument of the FROM_PROTOBUF SQL function
must be a constant " +
"map of strings to strings containing the options to use for
converting the value " +
"from Protobuf format"),
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
index 29351107a098..a34a2c740876 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
@@ -25,6 +25,7 @@ import scala.util.control.NonFatal
import org.apache.commons.io.FileUtils
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.util.ArrayBasedMapData
import org.apache.spark.sql.errors.QueryCompilationErrors
@@ -161,6 +162,9 @@ case class FromProtobuf(
val expr = constructor.newInstance(data, messageNameValue,
descFilePathValue, optionsValue)
expr.asInstanceOf[Expression]
}
+
+ override def prettyName: String =
+ getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("from_protobuf")
}
/**
@@ -272,4 +276,7 @@ case class ToProtobuf(
val expr = constructor.newInstance(data, messageNameValue,
descFilePathValue, optionsValue)
expr.asInstanceOf[Expression]
}
+
+ override def prettyName: String =
+ getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("to_protobuf")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
