This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 899153039023 [SPARK-47202][PYTHON][TESTS][FOLLOW-UP] Run the test only
with Python 3.9+
899153039023 is described below
commit 899153039023f2a12f43813028dcb54c9e880eda
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Thu Feb 29 10:56:26 2024 +0900
[SPARK-47202][PYTHON][TESTS][FOLLOW-UP] Run the test only with Python 3.9+
This PR proposes to run the tzinfo test only with Python 3.9+. This is a
followup of https://github.com/apache/spark/pull/45308.
To make the Python build passing with Python 3.8. It fails as below:
```python
Starting test(pypy3): pyspark.sql.tests.test_arrow (temp output:
/__w/spark/spark/python/target/605c2e61-b7c8-4898-ac7b-1d86f495bd4f/pypy3__pyspark.sql.tests.test_arrow__qrwyvw4l.log)
Traceback (most recent call last):
File "/usr/local/pypy/pypy3.8/lib/pypy3.8/runpy.py", line 197, in
_run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/local/pypy/pypy3.8/lib/pypy3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/__w/spark/spark/python/pyspark/sql/tests/test_arrow.py", line 26,
in <module>
from zoneinfo import ZoneInfo
ModuleNotFoundError: No module named 'zoneinfo'
```
https://github.com/apache/spark/actions/runs/8082492167/job/22083534905
No, test-only.
CI in this PR should test it out.
No,
Closes #45324 from HyukjinKwon/SPARK-47202-followup2.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit 11e8ae42af9234adf56dc2f32b92e87697c777e4)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/tests/connect/test_parity_arrow.py | 2 ++
python/pyspark/sql/tests/test_arrow.py | 5 ++++-
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/sql/tests/connect/test_parity_arrow.py
b/python/pyspark/sql/tests/connect/test_parity_arrow.py
index 55e689da8fdd..abcf839f0fc5 100644
--- a/python/pyspark/sql/tests/connect/test_parity_arrow.py
+++ b/python/pyspark/sql/tests/connect/test_parity_arrow.py
@@ -16,6 +16,7 @@
#
import unittest
+import sys
from distutils.version import LooseVersion
import pandas as pd
@@ -136,6 +137,7 @@ class ArrowParityTests(ArrowTestsMixin,
ReusedConnectTestCase, PandasOnSparkTest
def test_toPandas_nested_timestamp(self):
self.check_toPandas_nested_timestamp(True)
+ @unittest.skipIf(sys.version_info < (3, 9), "zoneinfo is available from
Python 3.9+")
def test_toPandas_timestmap_tzinfo(self):
self.check_toPandas_timestmap_tzinfo(True)
diff --git a/python/pyspark/sql/tests/test_arrow.py
b/python/pyspark/sql/tests/test_arrow.py
index 6e462d38d8e5..a333318b777e 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -25,7 +25,7 @@ import warnings
from distutils.version import LooseVersion
from typing import cast
from collections import namedtuple
-from zoneinfo import ZoneInfo
+import sys
from pyspark import SparkContext, SparkConf
from pyspark.sql import Row, SparkSession
@@ -1092,6 +1092,7 @@ class ArrowTestsMixin:
self.assertEqual(df.first(), expected)
+ @unittest.skipIf(sys.version_info < (3, 9), "zoneinfo is available from
Python 3.9+")
def test_toPandas_timestmap_tzinfo(self):
for arrow_enabled in [True, False]:
with self.subTest(arrow_enabled=arrow_enabled):
@@ -1099,6 +1100,8 @@ class ArrowTestsMixin:
def check_toPandas_timestmap_tzinfo(self, arrow_enabled):
# SPARK-47202: Test timestamp with tzinfo in toPandas and
createDataFrame
+ from zoneinfo import ZoneInfo
+
ts_tzinfo = datetime.datetime(2023, 1, 1, 0, 0, 0,
tzinfo=ZoneInfo("America/Los_Angeles"))
data = pd.DataFrame({"a": [ts_tzinfo]})
df = self.spark.createDataFrame(data)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
