Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/21107#discussion_r183932479
--- Diff: python/pyspark/sql/tests.py ---
@@ -3088,23 +3088,28 @@ def setUpClass(cls):
filename_pattern = (
"sql/core/target/scala-*/test-classes/org/apache/spark/sql/"
"TestQueryExecutionListener.class")
- if not glob.glob(os.path.join(SPARK_HOME, filename_pattern)):
- raise unittest.SkipTest(
+ cls.has_listener = bool(glob.glob(os.path.join(SPARK_HOME,
filename_pattern)))
+
+ if cls.has_listener:
+ # Note that 'spark.sql.queryExecutionListeners' is a static
immutable configuration.
+ cls.spark = SparkSession.builder \
+ .master("local[4]") \
+ .appName(cls.__name__) \
+ .config(
+ "spark.sql.queryExecutionListeners",
+ "org.apache.spark.sql.TestQueryExecutionListener") \
+ .getOrCreate()
+
+ def setUp(self):
+ if not self.has_listener:
+ raise self.skipTest(
"'org.apache.spark.sql.TestQueryExecutionListener' is not "
"available. Will skip the related tests.")
- # Note that 'spark.sql.queryExecutionListeners' is a static
immutable configuration.
- cls.spark = SparkSession.builder \
- .master("local[4]") \
- .appName(cls.__name__) \
- .config(
- "spark.sql.queryExecutionListeners",
- "org.apache.spark.sql.TestQueryExecutionListener") \
- .getOrCreate()
-
@classmethod
def tearDownClass(cls):
- cls.spark.stop()
+ if hasattr(cls, "spark"):
+ cls.spark.stop()
def tearDown(self):
self.spark._jvm.OnSuccessCall.clear()
--- End diff --
Yup.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
