This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new b5e645363ac [SPARK-41728][CONNECT][PYTHON] Implement `unwrap_udt` function b5e645363ac is described below commit b5e645363aca765e7c679949f4ecdd9f89db4f51 Author: Ruifeng Zheng <ruife...@apache.org> AuthorDate: Tue Dec 27 19:09:39 2022 +0900 [SPARK-41728][CONNECT][PYTHON] Implement `unwrap_udt` function ### What changes were proposed in this pull request? Implement `unwrap_udt` function ### Why are the changes needed? for API coverage ### Does this PR introduce _any_ user-facing change? yes ### How was this patch tested? no added test, since `unwrap_udt` [requires a UDT ](https://github.com/apache/spark/blob/bf4981fd4adfe96d3962e2e165c5a5d307a0033d/python/pyspark/ml/tests/test_linalg.py#L355-L366)which is not supported yet Closes #39234 from zhengruifeng/connect_unwrap_udt. Authored-by: Ruifeng Zheng <ruife...@apache.org> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../apache/spark/sql/connect/planner/SparkConnectPlanner.scala | 3 +++ python/pyspark/sql/connect/functions.py | 10 ++++++++++ python/pyspark/sql/functions.py | 2 ++ 3 files changed, 15 insertions(+) diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index 42cc96f35dc..1645eb2c381 100644 --- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -752,6 +752,9 @@ class SparkConnectPlanner(session: SparkSession) { case "hours" if fun.getArgumentsCount == 1 => Some(Hours(transformExpression(fun.getArguments(0)))) + case "unwrap_udt" if fun.getArgumentsCount == 1 => + Some(UnwrapUDT(transformExpression(fun.getArguments(0)))) + case _ => None } } diff --git a/python/pyspark/sql/connect/functions.py b/python/pyspark/sql/connect/functions.py index 3bc502cd99d..b7b78542bb8 100644 --- a/python/pyspark/sql/connect/functions.py +++ b/python/pyspark/sql/connect/functions.py @@ -2260,8 +2260,18 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column: sha2.__doc__ = pysparkfuncs.sha2.__doc__ +# User Defined Function + + def call_udf(udfName: str, *cols: "ColumnOrName") -> Column: return _invoke_function(udfName, *[_to_col(c) for c in cols]) call_udf.__doc__ = pysparkfuncs.call_udf.__doc__ + + +def unwrap_udt(col: "ColumnOrName") -> Column: + return _invoke_function("unwrap_udt", _to_col(col)) + + +unwrap_udt.__doc__ = pysparkfuncs.unwrap_udt.__doc__ diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index d02b2966190..c533ca7be6e 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -9939,6 +9939,8 @@ def unwrap_udt(col: "ColumnOrName") -> Column: .. versionadded:: 3.4.0 + .. versionchanged:: 3.4.0 + Support Spark Connect. """ return _invoke_function("unwrap_udt", _to_java_column(col)) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org