This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3c8c7ad92cdc [SPARK-46255][PYTHON][CONNECT] Support complex type ->
string conversion
3c8c7ad92cdc is described below
commit 3c8c7ad92cdc3ae989456d87b0332cb917da7e4e
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Dec 5 13:05:36 2023 +0800
[SPARK-46255][PYTHON][CONNECT] Support complex type -> string conversion
### What changes were proposed in this pull request?
Support complex type -> string conversion
### Why are the changes needed?
to support `list -> str` conversion
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #44171 from zhengruifeng/py_connect_str_conv.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/sql/connect/conversion.py | 16 ----------------
python/pyspark/sql/tests/test_types.py | 13 +++++++++++++
2 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/python/pyspark/sql/connect/conversion.py
b/python/pyspark/sql/connect/conversion.py
index 550978d02f85..fb5a2d4b17b1 100644
--- a/python/pyspark/sql/connect/conversion.py
+++ b/python/pyspark/sql/connect/conversion.py
@@ -222,22 +222,6 @@ class LocalDataToArrowConversion:
if value is None:
return None
else:
- # only atomic types are supported
- assert isinstance(
- value,
- (
- bool,
- int,
- float,
- str,
- bytes,
- bytearray,
- decimal.Decimal,
- datetime.date,
- datetime.datetime,
- datetime.timedelta,
- ),
- )
if isinstance(value, bool):
# To match the PySpark which convert bool to string in
# the JVM side (python.EvaluatePython.makeFromJava)
diff --git a/python/pyspark/sql/tests/test_types.py
b/python/pyspark/sql/tests/test_types.py
index b038cf6ce5ba..a07309d1dff9 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -507,6 +507,19 @@ class TypesTestsMixin:
self.assertEqual(1, row.asDict()["l"][0].a)
self.assertEqual(1.0, row.asDict()["d"]["key"].c)
+ def test_convert_list_to_str(self):
+ data = [[[123], 120]]
+ schema = StructType(
+ [
+ StructField("name", StringType(), True),
+ StructField("income", LongType(), True),
+ ]
+ )
+ df = self.spark.createDataFrame(data, schema)
+ self.assertEqual(df.schema, schema)
+ self.assertEqual(df.count(), 1)
+ self.assertEqual(df.head(), Row(name="[123]", income=120))
+
def test_udt(self):
from pyspark.sql.types import _parse_datatype_json_string,
_infer_type, _make_type_verifier
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
