This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b5e645363ac [SPARK-41728][CONNECT][PYTHON] Implement `unwrap_udt`
function
b5e645363ac is described below
commit b5e645363aca765e7c679949f4ecdd9f89db4f51
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Dec 27 19:09:39 2022 +0900
[SPARK-41728][CONNECT][PYTHON] Implement `unwrap_udt` function
### What changes were proposed in this pull request?
Implement `unwrap_udt` function
### Why are the changes needed?
for API coverage
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
no added test, since `unwrap_udt` [requires a UDT
](https://github.com/apache/spark/blob/bf4981fd4adfe96d3962e2e165c5a5d307a0033d/python/pyspark/ml/tests/test_linalg.py#L355-L366)which
is not supported yet
Closes #39234 from zhengruifeng/connect_unwrap_udt.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../apache/spark/sql/connect/planner/SparkConnectPlanner.scala | 3 +++
python/pyspark/sql/connect/functions.py | 10 ++++++++++
python/pyspark/sql/functions.py | 2 ++
3 files changed, 15 insertions(+)
diff --git
a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index 42cc96f35dc..1645eb2c381 100644
---
a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++
b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -752,6 +752,9 @@ class SparkConnectPlanner(session: SparkSession) {
case "hours" if fun.getArgumentsCount == 1 =>
Some(Hours(transformExpression(fun.getArguments(0))))
+ case "unwrap_udt" if fun.getArgumentsCount == 1 =>
+ Some(UnwrapUDT(transformExpression(fun.getArguments(0))))
+
case _ => None
}
}
diff --git a/python/pyspark/sql/connect/functions.py
b/python/pyspark/sql/connect/functions.py
index 3bc502cd99d..b7b78542bb8 100644
--- a/python/pyspark/sql/connect/functions.py
+++ b/python/pyspark/sql/connect/functions.py
@@ -2260,8 +2260,18 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column:
sha2.__doc__ = pysparkfuncs.sha2.__doc__
+# User Defined Function
+
+
def call_udf(udfName: str, *cols: "ColumnOrName") -> Column:
return _invoke_function(udfName, *[_to_col(c) for c in cols])
call_udf.__doc__ = pysparkfuncs.call_udf.__doc__
+
+
+def unwrap_udt(col: "ColumnOrName") -> Column:
+ return _invoke_function("unwrap_udt", _to_col(col))
+
+
+unwrap_udt.__doc__ = pysparkfuncs.unwrap_udt.__doc__
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index d02b2966190..c533ca7be6e 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -9939,6 +9939,8 @@ def unwrap_udt(col: "ColumnOrName") -> Column:
.. versionadded:: 3.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
"""
return _invoke_function("unwrap_udt", _to_java_column(col))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
