HyukjinKwon commented on code in PR #59362:
URL: https://github.com/apache/spark/pull/59362#discussion_r4243618218
##########
python/pyspark/sql/tests/pandas/test_converter.py:
##########
@@ -581,6 +587,67 @@ def test_converter_from_pandas_struct(self):
),
)
+ def test_converter_from_pandas_nested_timestamp(self):
+ tz = "America/Los_Angeles"
+ aware = datetime.datetime(2020, 1, 1, 12, tzinfo=datetime.timezone.utc)
+ naive = datetime.datetime(2020, 1, 1, 12)
+ # The naive value is read in the session timezone: 12:00 PST (UTC-8)
-> 20:00 UTC.
+ naive_as_session = datetime.datetime(2020, 1, 1, 20,
tzinfo=datetime.timezone.utc)
+
+ conv = _create_converter_from_pandas(ArrayType(TimestampType()),
timezone=tz)
+ assert_series_equal(
+ conv(pd.Series([[aware, naive, None], None])),
+ pd.Series([[aware, naive_as_session, None], None]),
+ )
+
+ nanos = pd.Timestamp("2020-01-01 12:00:00.000000001", tz="UTC")
+ conv =
_create_converter_from_pandas(ArrayType(TimestampLTZNanosType()), timezone=tz)
+ result = conv(pd.Series([[nanos, naive, None], None]))
+ assert_series_equal(
+ result,
+ pd.Series([[nanos, pd.Timestamp(naive_as_session), None], None]),
+ )
+ self.assertEqual(result[0][0].nanosecond, 1)
+
+ def test_converter_to_pandas_nested_timestamp(self):
+ tz = "America/Los_Angeles"
+ aware = datetime.datetime(2020, 1, 1, 12, tzinfo=datetime.timezone.utc)
+ naive = datetime.datetime(2020, 1, 1, 12)
+ # 12:00 UTC -> 04:00 PST (UTC-8).
+ expected_from_utc = pd.Timestamp("2020-01-01 04:00:00")
+ # 12:00 JST (UTC+9) -> 03:00 UTC -> 19:00 PST on the previous day.
+ expected_from_local = pd.Timestamp("2019-12-31 19:00:00")
+
+ conv = _create_converter_to_pandas(ArrayType(TimestampType()),
timezone=tz)
+ assert_series_equal(
+ conv(pd.Series([[aware, naive, None], None])),
+ pd.Series([[expected_from_utc, expected_from_utc, None], None]),
+ )
+
+ with mock.patch.dict(os.environ, {"TZ": "Asia/Tokyo"}):
+ conv = _create_converter_to_pandas(
+ ArrayType(TimestampType()), timezone=tz,
timestamp_utc_localized=False
+ )
+ assert_series_equal(
+ conv(pd.Series([[aware, naive], None])),
+ pd.Series([[expected_from_utc, expected_from_local], None]),
+ )
+
+ nanos = pd.Timestamp("2020-01-01 12:00:00.000000001", tz="UTC")
+ conv = _create_converter_to_pandas(ArrayType(TimestampLTZNanosType()),
timezone=tz)
+ result = conv(pd.Series([[nanos, None], None]))
+ assert_series_equal(
+ result,
+ pd.Series([[pd.Timestamp("2020-01-01 04:00:00.000000001"), None],
None]),
+ )
+ self.assertEqual(result[0][0].nanosecond, 1)
+
+ conv = _create_converter_to_pandas(ArrayType(TimestampNTZType()))
+ assert_series_equal(
Review Comment:
**Nit (P3):** This check doesn't fail when the conversion it covers is
broken. `convert_timestamp_ntz` only wraps each element in
`pd.Timestamp(value)`, and `pd.Timestamp(naive) == naive` is `True`.
`assert_series_equal` on an object-dtype Series compares the nested cells by
equality, not by type. So if the NTZ branch of `_converter` were removed (the
array converter then falls back to identity), or returned the value unchanged,
the cell would be `[datetime.datetime(2020, 1, 1, 12), None]` and this
assertion would still pass.
Could you keep the result and also assert the element type, the same way the
LTZ-nanos cases above check `.nanosecond` directly? For example:
```python
result = conv(pd.Series([[naive, None], None]))
assert_series_equal(result, pd.Series([[pd.Timestamp(naive), None], None]))
self.assertIsInstance(result[0][0], pd.Timestamp)
```
**Verification:**
- **Regression:** If the nested NTZ element conversion is bypassed so the
converter returns the input unchanged, the strengthened NTZ check fails on the
element type. With the current converter it passes.
- **Behavior:** The pandas converter unit tests, including both new
nested-timestamp tests, pass against the current converter implementation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
