This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new bf0d3c5a027 [SPARK-42073][CONNECT][PYTHON][TESTS] Enable tests in
common/test_parity_serde, common/test_parity_types
bf0d3c5a027 is described below
commit bf0d3c5a02758116f820f50e46dd74d46330915e
Author: Sandeep Singh <[email protected]>
AuthorDate: Mon Jan 16 09:48:10 2023 +0900
[SPARK-42073][CONNECT][PYTHON][TESTS] Enable tests in
common/test_parity_serde, common/test_parity_types
### What changes were proposed in this pull request?
Enable tests in common/test_parity_serde, common/test_parity_types
### Why are the changes needed?
for test coverage
### Does this PR introduce _any_ user-facing change?
no, test-only
### How was this patch tested?
enabled UT
Closes #39583 from techaddict/SPARK-42073.
Authored-by: Sandeep Singh <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/tests/connect/test_parity_serde.py | 4 ----
python/pyspark/sql/tests/connect/test_parity_types.py | 8 --------
python/pyspark/sql/tests/test_serde.py | 2 +-
python/pyspark/sql/tests/test_types.py | 5 ++---
4 files changed, 3 insertions(+), 16 deletions(-)
diff --git a/python/pyspark/sql/tests/connect/test_parity_serde.py
b/python/pyspark/sql/tests/connect/test_parity_serde.py
index 7d8acdd2075..dc9b437e973 100644
--- a/python/pyspark/sql/tests/connect/test_parity_serde.py
+++ b/python/pyspark/sql/tests/connect/test_parity_serde.py
@@ -30,10 +30,6 @@ class SerdeParityTests(SerdeTestsMixin,
ReusedConnectTestCase):
def test_serialize_nested_array_and_map(self):
super().test_serialize_nested_array_and_map()
- @unittest.skip("Spark Connect does not support RDD but the tests depend on
them.")
- def test_struct_in_map(self):
- super().test_struct_in_map()
-
if __name__ == "__main__":
import unittest
diff --git a/python/pyspark/sql/tests/connect/test_parity_types.py
b/python/pyspark/sql/tests/connect/test_parity_types.py
index ea1d6dc658c..38c5ff61329 100644
--- a/python/pyspark/sql/tests/connect/test_parity_types.py
+++ b/python/pyspark/sql/tests/connect/test_parity_types.py
@@ -59,10 +59,6 @@ class TypesParityTests(TypesTestsMixin,
ReusedConnectTestCase):
def test_complex_nested_udt_in_df(self):
super().test_complex_nested_udt_in_df()
- @unittest.skip("Spark Connect does not support RDD but the tests depend on
them.")
- def test_convert_row_to_dict(self):
- super().test_convert_row_to_dict()
-
# TODO(SPARK-42020): createDataFrame with UDT
@unittest.skip("Fails in Spark Connect, should enable.")
def test_create_dataframe_from_objects(self):
@@ -137,10 +133,6 @@ class TypesParityTests(TypesTestsMixin,
ReusedConnectTestCase):
def test_infer_schema_with_udt(self):
super().test_infer_schema_with_udt()
- @unittest.skip("Spark Connect does not support RDD but the tests depend on
them.")
- def test_metadata_null(self):
- super().test_metadata_null()
-
# TODO(SPARK-41834): Implement SparkSession.conf
@unittest.skip("Fails in Spark Connect, should enable.")
def test_negative_decimal(self):
diff --git a/python/pyspark/sql/tests/test_serde.py
b/python/pyspark/sql/tests/test_serde.py
index f24a716f4a6..ef8bbd2c370 100644
--- a/python/pyspark/sql/tests/test_serde.py
+++ b/python/pyspark/sql/tests/test_serde.py
@@ -54,7 +54,7 @@ class SerdeTestsMixin:
def test_struct_in_map(self):
d = [Row(m={Row(i=1): Row(s="")})]
- df = self.sc.parallelize(d).toDF()
+ df = self.spark.createDataFrame(d)
k, v = list(df.head().m.items())[0]
self.assertEqual(1, k.i)
self.assertEqual("", v.s)
diff --git a/python/pyspark/sql/tests/test_types.py
b/python/pyspark/sql/tests/test_types.py
index 392b1f95d8b..a4d444830e7 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -480,7 +480,7 @@ class TypesTestsMixin:
def test_convert_row_to_dict(self):
row = Row(l=[Row(a=1, b="s")], d={"key": Row(c=1.0, d="2")})
self.assertEqual(1, row.asDict()["l"][0].a)
- df = self.sc.parallelize([row]).toDF()
+ df = self.spark.createDataFrame([row])
with self.tempView("test"):
df.createOrReplaceTempView("test")
@@ -788,8 +788,7 @@ class TypesTestsMixin:
StructField("f2", StringType(), True, {"a": None}),
]
)
- rdd = self.sc.parallelize([["a", "b"], ["c", "d"]])
- self.spark.createDataFrame(rdd, schema)
+ self.spark.createDataFrame([["a", "b"], ["c", "d"]], schema)
def test_access_nested_types(self):
df = self.spark.createDataFrame([Row(l=[1], r=Row(a=1, b="b"), d={"k":
"v"})])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
