This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 0c9b1232641 [SPARK-41723][CONNECT][PYTHON] Implement `sequence`
function
0c9b1232641 is described below
commit 0c9b123264161202950c1a09a0606ddf57a00635
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Dec 27 14:14:05 2022 +0800
[SPARK-41723][CONNECT][PYTHON] Implement `sequence` function
### What changes were proposed in this pull request?
Implement `sequence` function
### Why are the changes needed?
for API coverage
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
added ut
Closes #39228 from zhengruifeng/connect_function_sequence.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/sql/connect/functions.py | 12 ++++++++++++
.../pyspark/sql/tests/connect/test_connect_function.py | 18 ++++++++++++++++++
2 files changed, 30 insertions(+)
diff --git a/python/pyspark/sql/connect/functions.py
b/python/pyspark/sql/connect/functions.py
index ad255266b21..0ed3629751d 100644
--- a/python/pyspark/sql/connect/functions.py
+++ b/python/pyspark/sql/connect/functions.py
@@ -1414,6 +1414,18 @@ def reverse(col: "ColumnOrName") -> Column:
reverse.__doc__ = pysparkfuncs.reverse.__doc__
+def sequence(
+ start: "ColumnOrName", stop: "ColumnOrName", step:
Optional["ColumnOrName"] = None
+) -> Column:
+ if step is None:
+ return _invoke_function_over_columns("sequence", start, stop)
+ else:
+ return _invoke_function_over_columns("sequence", start, stop, step)
+
+
+sequence.__doc__ = pysparkfuncs.sequence.__doc__
+
+
# TODO(SPARK-41493): Support options
def schema_of_csv(csv: "ColumnOrName") -> Column:
if isinstance(csv, Column):
diff --git a/python/pyspark/sql/tests/connect/test_connect_function.py
b/python/pyspark/sql/tests/connect/test_connect_function.py
index 37c29e2e66b..40d8c924f60 100644
--- a/python/pyspark/sql/tests/connect/test_connect_function.py
+++ b/python/pyspark/sql/tests/connect/test_connect_function.py
@@ -1049,6 +1049,24 @@ class
SparkConnectFunctionTests(SparkConnectFuncTestCase):
sdf.select(SF.struct(sdf.a, "d", "e", sdf.f)),
)
+ # test sequence
+ self.assert_eq(
+ cdf.select(CF.sequence(CF.lit(1), CF.lit(5))).toPandas(),
+ sdf.select(SF.sequence(SF.lit(1), SF.lit(5))).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(CF.sequence(CF.lit(1), CF.lit(5),
CF.lit(1))).toPandas(),
+ sdf.select(SF.sequence(SF.lit(1), SF.lit(5),
SF.lit(1))).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(CF.sequence(cdf.d, "e")).toPandas(),
+ sdf.select(SF.sequence(sdf.d, "e")).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(CF.sequence(cdf.d, "e", CF.lit(1))).toPandas(),
+ sdf.select(SF.sequence(sdf.d, "e", SF.lit(1))).toPandas(),
+ )
+
def test_map_collection_functions(self):
from pyspark.sql import functions as SF
from pyspark.sql.connect import functions as CF
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
