Xiugapurin commented on code in PR #59362:
URL: https://github.com/apache/spark/pull/59362#discussion_r4254219168
##########
python/pyspark/sql/tests/pandas/test_converter.py:
##########
@@ -581,6 +587,67 @@ def test_converter_from_pandas_struct(self):
),
)
+ def test_converter_from_pandas_nested_timestamp(self):
+ tz = "America/Los_Angeles"
+ aware = datetime.datetime(2020, 1, 1, 12, tzinfo=datetime.timezone.utc)
+ naive = datetime.datetime(2020, 1, 1, 12)
+ # The naive value is read in the session timezone: 12:00 PST (UTC-8)
-> 20:00 UTC.
+ naive_as_session = datetime.datetime(2020, 1, 1, 20,
tzinfo=datetime.timezone.utc)
+
+ conv = _create_converter_from_pandas(ArrayType(TimestampType()),
timezone=tz)
+ assert_series_equal(
+ conv(pd.Series([[aware, naive, None], None])),
+ pd.Series([[aware, naive_as_session, None], None]),
+ )
+
+ nanos = pd.Timestamp("2020-01-01 12:00:00.000000001", tz="UTC")
+ conv =
_create_converter_from_pandas(ArrayType(TimestampLTZNanosType()), timezone=tz)
+ result = conv(pd.Series([[nanos, naive, None], None]))
+ assert_series_equal(
+ result,
+ pd.Series([[nanos, pd.Timestamp(naive_as_session), None], None]),
+ )
+ self.assertEqual(result[0][0].nanosecond, 1)
+
+ def test_converter_to_pandas_nested_timestamp(self):
+ tz = "America/Los_Angeles"
+ aware = datetime.datetime(2020, 1, 1, 12, tzinfo=datetime.timezone.utc)
+ naive = datetime.datetime(2020, 1, 1, 12)
+ # 12:00 UTC -> 04:00 PST (UTC-8).
+ expected_from_utc = pd.Timestamp("2020-01-01 04:00:00")
+ # 12:00 JST (UTC+9) -> 03:00 UTC -> 19:00 PST on the previous day.
+ expected_from_local = pd.Timestamp("2019-12-31 19:00:00")
+
+ conv = _create_converter_to_pandas(ArrayType(TimestampType()),
timezone=tz)
+ assert_series_equal(
+ conv(pd.Series([[aware, naive, None], None])),
+ pd.Series([[expected_from_utc, expected_from_utc, None], None]),
+ )
+
+ with mock.patch.dict(os.environ, {"TZ": "Asia/Tokyo"}):
+ conv = _create_converter_to_pandas(
+ ArrayType(TimestampType()), timezone=tz,
timestamp_utc_localized=False
+ )
+ assert_series_equal(
+ conv(pd.Series([[aware, naive], None])),
+ pd.Series([[expected_from_utc, expected_from_local], None]),
+ )
+
+ nanos = pd.Timestamp("2020-01-01 12:00:00.000000001", tz="UTC")
+ conv = _create_converter_to_pandas(ArrayType(TimestampLTZNanosType()),
timezone=tz)
+ result = conv(pd.Series([[nanos, None], None]))
+ assert_series_equal(
+ result,
+ pd.Series([[pd.Timestamp("2020-01-01 04:00:00.000000001"), None],
None]),
+ )
+ self.assertEqual(result[0][0].nanosecond, 1)
+
+ conv = _create_converter_to_pandas(ArrayType(TimestampNTZType()))
+ assert_series_equal(
Review Comment:
Thanks for the review! Updated to keep the result and assert the element
type is `pd.Timestamp`. I also verified that the strengthened check fails when
the NTZ conversion is bypassed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
