This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 49da3a43f7ab [SPARK-48340][PYTHON][FOLLOWUP] Support TimestampNTZ
infer schema miss prefer_timestamp_ntz
49da3a43f7ab is described below
commit 49da3a43f7ab41dad59e7deb810974902625c41c
Author: Angerszhuuuu <[email protected]>
AuthorDate: Mon May 27 17:32:46 2024 +0900
[SPARK-48340][PYTHON][FOLLOWUP] Support TimestampNTZ infer schema miss
prefer_timestamp_ntz
### What changes were proposed in this pull request?
Add UT
### Why are the changes needed?
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added UT
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #46750 from AngersZhuuuu/SPARK-48340-FOLLOWUP.
Lead-authored-by: Angerszhuuuu <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/tests/test_types.py | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/sql/tests/test_types.py
b/python/pyspark/sql/tests/test_types.py
index d665053d9490..80f2c0fcbc03 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -192,6 +192,7 @@ class TypesTestsMixin:
Row(a=1),
Row("a")(1),
A(),
+ Row(b=Row(c=datetime.datetime(1970, 1, 1, 0, 0))),
]
df = self.spark.createDataFrame([data])
@@ -214,6 +215,7 @@ class TypesTestsMixin:
"struct<a:bigint>",
"struct<a:bigint>",
"struct<a:bigint>",
+ "struct<b:struct<c:timestamp>>",
]
self.assertEqual(actual, expected)
@@ -236,14 +238,25 @@ class TypesTestsMixin:
Row(a=1),
Row(a=1),
Row(a=1),
+ Row(b=Row(c=datetime.datetime(1970, 1, 1, 0, 0))),
]
self.assertEqual(actual, expected)
with self.sql_conf({"spark.sql.timestampType": "TIMESTAMP_NTZ"}):
with self.sql_conf({"spark.sql.session.timeZone":
"America/Sao_Paulo"}):
- df = self.spark.createDataFrame([(datetime.datetime(1970, 1,
1, 0, 0),)])
+ data = [
+ (
+ datetime.datetime(1970, 1, 1, 0, 0),
+ Row(a=Row(a=datetime.datetime(1970, 1, 1, 0, 0))),
+ )
+ ]
+ df = self.spark.createDataFrame(data)
self.assertEqual(list(df.schema)[0].dataType.simpleString(),
"timestamp_ntz")
self.assertEqual(df.first()[0], datetime.datetime(1970, 1, 1,
0, 0))
+ self.assertEqual(
+ list(df.schema)[1].dataType.simpleString(),
"struct<a:struct<a:timestamp_ntz>>"
+ )
+ self.assertEqual(df.first()[1],
Row(a=Row(a=datetime.datetime(1970, 1, 1, 0, 0))))
df = self.spark.createDataFrame(
[
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
