This is an automated email from the ASF dual-hosted git repository.
xinrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 29a70117b27 [SPARK-41225][CONNECT][PYTHON][FOLLOW-UP] Disable
unsupported functions
29a70117b27 is described below
commit 29a70117b272582d11e7b7b8951dff1be91d3de7
Author: Martin Grund <martin.gr...@databricks.com>
AuthorDate: Fri Dec 9 14:55:50 2022 -0800
[SPARK-41225][CONNECT][PYTHON][FOLLOW-UP] Disable unsupported functions
### What changes were proposed in this pull request?
This patch adds method stubs for unsupported functions in the Python client
for Spark Connect in the `Column` class that will throw a
`NoteImplementedError` when called. This is to give a clear indication to the
users that these methods will be implemented in the future.
### Why are the changes needed?
UX
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
UT
Closes #39009 from grundprinzip/SPARK-41225-v2.
Authored-by: Martin Grund <martin.gr...@databricks.com>
Signed-off-by: Xinrong Meng <xinr...@apache.org>
---
python/pyspark/sql/connect/column.py | 36 ++++++++++++++++++++++
.../sql/tests/connect/test_connect_column.py | 25 +++++++++++++++
2 files changed, 61 insertions(+)
diff --git a/python/pyspark/sql/connect/column.py
b/python/pyspark/sql/connect/column.py
index 63e95c851db..f1a909b89fc 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -786,3 +786,39 @@ class Column:
def __repr__(self) -> str:
return "Column<'%s'>" % self._expr.__repr__()
+
+ def otherwise(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("otherwise() is not yet implemented.")
+
+ def over(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("over() is not yet implemented.")
+
+ def isin(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("isin() is not yet implemented.")
+
+ def when(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("when() is not yet implemented.")
+
+ def getItem(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("getItem() is not yet implemented.")
+
+ def astype(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("astype() is not yet implemented.")
+
+ def between(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("between() is not yet implemented.")
+
+ def getField(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("getField() is not yet implemented.")
+
+ def withField(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("withField() is not yet implemented.")
+
+ def dropFields(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("dropFields() is not yet implemented.")
+
+ def __getitem__(self, k: Any) -> None:
+ raise NotImplementedError("apply() - __getitem__ is not yet
implemented.")
+
+ def __iter__(self) -> None:
+ raise TypeError("Column is not iterable")
diff --git a/python/pyspark/sql/tests/connect/test_connect_column.py
b/python/pyspark/sql/tests/connect/test_connect_column.py
index c73f1b5b0c7..734b0bbf226 100644
--- a/python/pyspark/sql/tests/connect/test_connect_column.py
+++ b/python/pyspark/sql/tests/connect/test_connect_column.py
@@ -119,6 +119,31 @@ class SparkConnectTests(SparkConnectSQLTestCase):
df.select(df.id.cast(x)).toPandas(),
df2.select(df2.id.cast(x)).toPandas()
)
+ def test_unsupported_functions(self):
+ # SPARK-41225: Disable unsupported functions.
+ c = self.connect.range(1).id
+ for f in (
+ "otherwise",
+ "over",
+ "isin",
+ "when",
+ "getItem",
+ "astype",
+ "between",
+ "getField",
+ "withField",
+ "dropFields",
+ ):
+ with self.assertRaises(NotImplementedError):
+ getattr(c, f)()
+
+ with self.assertRaises(NotImplementedError):
+ c["a"]
+
+ with self.assertRaises(TypeError):
+ for x in c:
+ pass
+
if __name__ == "__main__":
import unittest
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
