This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new cd618822796 [SPARK-41964][CONNECT][PYTHON] Add the list of unsupported
IO functions
cd618822796 is described below
commit cd61882279694aa71f0ee1c717387e289697e4c7
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Wed Jan 11 09:22:52 2023 +0900
[SPARK-41964][CONNECT][PYTHON] Add the list of unsupported IO functions
### What changes were proposed in this pull request?
Add the list of unsupported IO functions
### Why are the changes needed?
missing APIs should throw `NotImplementedError` other than `AttributeError`
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
added UT
Closes #39484 from zhengruifeng/connect_io_missing.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/connect/readwriter.py | 9 +++++++++
python/pyspark/sql/tests/connect/test_connect_basic.py | 12 ++++++++++++
2 files changed, 21 insertions(+)
diff --git a/python/pyspark/sql/connect/readwriter.py
b/python/pyspark/sql/connect/readwriter.py
index 80cecbc7f50..f256e5d3118 100644
--- a/python/pyspark/sql/connect/readwriter.py
+++ b/python/pyspark/sql/connect/readwriter.py
@@ -221,6 +221,15 @@ class DataFrameReader(OptionUtils):
text.__doc__ = PySparkDataFrameReader.text.__doc__
+ def csv(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("csv() is not implemented.")
+
+ def orc(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("orc() is not implemented.")
+
+ def jdbc(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("jdbc() not supported for DataFrameWriter")
+
DataFrameReader.__doc__ = PySparkDataFrameReader.__doc__
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py
b/python/pyspark/sql/tests/connect/test_connect_basic.py
index d8bde3db6d7..2b03ce70638 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -2163,6 +2163,18 @@ class SparkConnectBasicTests(SparkConnectSQLTestCase):
with self.assertRaises(NotImplementedError):
getattr(self.connect.catalog, f)()
+ def test_unsupported_io_functions(self):
+ # SPARK-41964: Disable unsupported functions.
+ # DataFrameWriterV2 is also not implemented yet
+
+ for f in ("csv", "orc", "jdbc"):
+ with self.assertRaises(NotImplementedError):
+ getattr(self.connect.read, f)()
+
+ for f in ("jdbc",):
+ with self.assertRaises(NotImplementedError):
+ getattr(self.connect.read, f)()
+
@unittest.skipIf(not should_test_connect, connect_requirement_message)
class ChannelBuilderTests(ReusedPySparkTestCase):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
