This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e6c97ebf3f0f [SPARK-54337][PS][TESTS][FOLLOWUP] Fix
test_arrow_c_stream when pandas/arrow does not exist
e6c97ebf3f0f is described below
commit e6c97ebf3f0fc0c19e858165c5c28082b0d0311d
Author: Tian Gao <[email protected]>
AuthorDate: Sat Jan 17 09:04:12 2026 +0800
[SPARK-54337][PS][TESTS][FOLLOWUP] Fix test_arrow_c_stream when
pandas/arrow does not exist
### What changes were proposed in this pull request?
Like all other tests, a test can't just `import pyarrow` or `import pandas`.
### Why are the changes needed?
https://github.com/apache/spark/actions/runs/21035972618/job/60484152696
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI should continue to pass and we need to wait for the daily scheduled CI.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53825 from gaogaotiantian/fix-cstreamer-test.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../pyspark/sql/tests/arrow/test_arrow_c_stream.py | 28 ++++++++++++++--------
1 file changed, 18 insertions(+), 10 deletions(-)
diff --git a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py
b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py
index 9534db71bae6..e71d1d18329f 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py
@@ -16,11 +16,25 @@
#
import ctypes
import unittest
-import pyarrow as pa
-import pandas as pd
import pyspark.pandas as ps
+from pyspark.testing.utils import (
+ have_pandas,
+ have_pyarrow,
+ pandas_requirement_message,
+ pyarrow_requirement_message,
+)
+if have_pandas:
+ import pandas as pd
+if have_pyarrow:
+ import pyarrow as pa
+
+
[email protected](
+ not have_pyarrow or not have_pandas,
+ pyarrow_requirement_message or pandas_requirement_message,
+)
class TestSparkArrowCStreamer(unittest.TestCase):
def test_spark_arrow_c_streamer_arrow_consumer(self):
pdf = pd.DataFrame([[1, "a"], [2, "b"], [3, "c"], [4, "d"]],
columns=["id", "value"])
@@ -53,12 +67,6 @@ class TestSparkArrowCStreamer(unittest.TestCase):
if __name__ == "__main__":
- from pyspark.sql.tests.arrow.test_arrow_c_stream import * # noqa: F401
-
- try:
- import xmlrunner # type: ignore
+ from pyspark.testing import main
- test_runner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
- except ImportError:
- test_runner = None
- unittest.main(testRunner=test_runner, verbosity=2)
+ main()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
