This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9d52342d2d6 [SPARK-41346][CONNECT][PYTHON][FOLLOWUP]
test_connect_function` cleanup
9d52342d2d6 is described below
commit 9d52342d2d69a55f2eda718d3e7f6b10f60e41ab
Author: Ruifeng Zheng <ruife...@apache.org>
AuthorDate: Mon Dec 5 11:02:12 2022 +0800
[SPARK-41346][CONNECT][PYTHON][FOLLOWUP] test_connect_function` cleanup
### What changes were proposed in this pull request?
remove unused tables, variables in `test_connect_function`
### Why are the changes needed?
`test_connect_function` does not need a saved table at all
### Does this PR introduce _any_ user-facing change?
No, test only
### How was this patch tested?
updated test
Closes #38886 from zhengruifeng/function_test_cleanup.
Lead-authored-by: Ruifeng Zheng <ruife...@apache.org>
Co-authored-by: Hyukjin Kwon <gurwls...@gmail.com>
Signed-off-by: Ruifeng Zheng <ruife...@apache.org>
---
.../sql/tests/connect/test_connect_function.py | 46 +++-------------------
1 file changed, 5 insertions(+), 41 deletions(-)
diff --git a/python/pyspark/sql/tests/connect/test_connect_function.py
b/python/pyspark/sql/tests/connect/test_connect_function.py
index f2c9b188985..41b404edcab 100644
--- a/python/pyspark/sql/tests/connect/test_connect_function.py
+++ b/python/pyspark/sql/tests/connect/test_connect_function.py
@@ -20,8 +20,7 @@ import tempfile
from pyspark.testing.sqlutils import have_pandas, SQLTestUtils
-from pyspark.sql import SparkSession, Row
-from pyspark.sql.types import StructType, StructField, StringType
+from pyspark.sql import SparkSession
if have_pandas:
from pyspark.sql.connect.session import SparkSession as RemoteSparkSession
@@ -32,7 +31,7 @@ from pyspark.testing.utils import ReusedPySparkTestCase
@unittest.skipIf(not should_test_connect, connect_requirement_message)
-class SparkConnectSQLTestCase(PandasOnSparkTestCase, ReusedPySparkTestCase,
SQLTestUtils):
+class SparkConnectFuncTestCase(PandasOnSparkTestCase, ReusedPySparkTestCase,
SQLTestUtils):
"""Parent test fixture class for all Spark Connect related
test cases."""
@@ -50,50 +49,15 @@ class SparkConnectSQLTestCase(PandasOnSparkTestCase,
ReusedPySparkTestCase, SQLT
cls.hive_available = True
# Create the new Spark Session
cls.spark = SparkSession(cls.sc)
- cls.testData = [Row(key=i, value=str(i)) for i in range(100)]
- cls.testDataStr = [Row(key=str(i)) for i in range(100)]
- cls.df = cls.sc.parallelize(cls.testData).toDF()
- cls.df_text = cls.sc.parallelize(cls.testDataStr).toDF()
-
- cls.tbl_name = "test_connect_basic_table_1"
- cls.tbl_name_empty = "test_connect_basic_table_empty"
-
- # Cleanup test data
- cls.spark_connect_clean_up_test_data()
- # Load test data
- cls.spark_connect_load_test_data()
-
- @classmethod
- def tearDownClass(cls: Any) -> None:
- cls.spark_connect_clean_up_test_data()
- ReusedPySparkTestCase.tearDownClass()
-
- @classmethod
- def spark_connect_load_test_data(cls: Any):
# Setup Remote Spark Session
cls.connect = RemoteSparkSession.builder.remote().getOrCreate()
- df = cls.spark.createDataFrame([(x, f"{x}") for x in range(100)],
["id", "name"])
- # Since we might create multiple Spark sessions, we need to create
global temporary view
- # that is specifically maintained in the "global_temp" schema.
- df.write.saveAsTable(cls.tbl_name)
- empty_table_schema = StructType(
- [
- StructField("firstname", StringType(), True),
- StructField("middlename", StringType(), True),
- StructField("lastname", StringType(), True),
- ]
- )
- emptyRDD = cls.spark.sparkContext.emptyRDD()
- empty_df = cls.spark.createDataFrame(emptyRDD, empty_table_schema)
- empty_df.write.saveAsTable(cls.tbl_name_empty)
@classmethod
- def spark_connect_clean_up_test_data(cls: Any) -> None:
- cls.spark.sql("DROP TABLE IF EXISTS {}".format(cls.tbl_name))
- cls.spark.sql("DROP TABLE IF EXISTS {}".format(cls.tbl_name_empty))
+ def tearDownClass(cls: Any) -> None:
+ ReusedPySparkTestCase.tearDownClass()
-class SparkConnectFunctionTests(SparkConnectSQLTestCase):
+class SparkConnectFunctionTests(SparkConnectFuncTestCase):
"""These test cases exercise the interface to the proto plan
generation but do not call Spark."""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
