This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b614fbef772 [SPARK-41283][CONNECT][PYTHON] Add `array_append` to
Connect
b614fbef772 is described below
commit b614fbef7726b38ef90f46e00e326fd9c9f91c45
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Sun Jan 22 17:50:14 2023 +0800
[SPARK-41283][CONNECT][PYTHON] Add `array_append` to Connect
### What changes were proposed in this pull request?
`array_append` was recently added in SQL and PySpark, this PR adds it to
Connect.
### Why are the changes needed?
For parity
### Does this PR introduce _any_ user-facing change?
yes, new function
### How was this patch tested?
added test
Closes #39698 from zhengruifeng/connect_function_array_append.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/sql/connect/functions.py | 7 +++++++
python/pyspark/sql/functions.py | 3 +++
python/pyspark/sql/tests/connect/test_connect_function.py | 10 ++++++++++
3 files changed, 20 insertions(+)
diff --git a/python/pyspark/sql/connect/functions.py
b/python/pyspark/sql/connect/functions.py
index a2e3f0c35b3..75f6ba1ff64 100644
--- a/python/pyspark/sql/connect/functions.py
+++ b/python/pyspark/sql/connect/functions.py
@@ -1140,6 +1140,13 @@ def array(*cols: Union["ColumnOrName",
List["ColumnOrName"], Tuple["ColumnOrName
array.__doc__ = pysparkfuncs.array.__doc__
+def array_append(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("array_append", col1, col2)
+
+
+array_append.__doc__ = pysparkfuncs.array_append.__doc__
+
+
def array_contains(col: "ColumnOrName", value: Any) -> Column:
return _invoke_function("array_contains", _to_col(col), lit(value))
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 70e74a92fab..2c025db0f36 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -7812,6 +7812,9 @@ def array_append(col1: "ColumnOrName", col2:
"ColumnOrName") -> Column:
.. versionadded:: 3.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col1 : :class:`~pyspark.sql.Column` or str
diff --git a/python/pyspark/sql/tests/connect/test_connect_function.py
b/python/pyspark/sql/tests/connect/test_connect_function.py
index fe381eb7d35..79812a945f1 100644
--- a/python/pyspark/sql/tests/connect/test_connect_function.py
+++ b/python/pyspark/sql/tests/connect/test_connect_function.py
@@ -1070,6 +1070,16 @@ class
SparkConnectFunctionTests(SparkConnectFuncTestCase):
sdf.select(SF.array_contains(sdf.a, sdf.f)).toPandas(),
)
+ # test array_append
+ self.assert_eq(
+ cdf.select(CF.array_append(cdf.a, CF.lit("ab"))).toPandas(),
+ sdf.select(SF.array_append(sdf.a, SF.lit("ab"))).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(CF.array_append(cdf.a, cdf.f)).toPandas(),
+ sdf.select(SF.array_append(sdf.a, sdf.f)).toPandas(),
+ )
+
# test array_join
self.assert_eq(
cdf.select(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
