This is an automated email from the ASF dual-hosted git repository. ruifengz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 9d52342d2d6 [SPARK-41346][CONNECT][PYTHON][FOLLOWUP] test_connect_function` cleanup 9d52342d2d6 is described below commit 9d52342d2d69a55f2eda718d3e7f6b10f60e41ab Author: Ruifeng Zheng <ruife...@apache.org> AuthorDate: Mon Dec 5 11:02:12 2022 +0800 [SPARK-41346][CONNECT][PYTHON][FOLLOWUP] test_connect_function` cleanup ### What changes were proposed in this pull request? remove unused tables, variables in `test_connect_function` ### Why are the changes needed? `test_connect_function` does not need a saved table at all ### Does this PR introduce _any_ user-facing change? No, test only ### How was this patch tested? updated test Closes #38886 from zhengruifeng/function_test_cleanup. Lead-authored-by: Ruifeng Zheng <ruife...@apache.org> Co-authored-by: Hyukjin Kwon <gurwls...@gmail.com> Signed-off-by: Ruifeng Zheng <ruife...@apache.org> --- .../sql/tests/connect/test_connect_function.py | 46 +++------------------- 1 file changed, 5 insertions(+), 41 deletions(-) diff --git a/python/pyspark/sql/tests/connect/test_connect_function.py b/python/pyspark/sql/tests/connect/test_connect_function.py index f2c9b188985..41b404edcab 100644 --- a/python/pyspark/sql/tests/connect/test_connect_function.py +++ b/python/pyspark/sql/tests/connect/test_connect_function.py @@ -20,8 +20,7 @@ import tempfile from pyspark.testing.sqlutils import have_pandas, SQLTestUtils -from pyspark.sql import SparkSession, Row -from pyspark.sql.types import StructType, StructField, StringType +from pyspark.sql import SparkSession if have_pandas: from pyspark.sql.connect.session import SparkSession as RemoteSparkSession @@ -32,7 +31,7 @@ from pyspark.testing.utils import ReusedPySparkTestCase @unittest.skipIf(not should_test_connect, connect_requirement_message) -class SparkConnectSQLTestCase(PandasOnSparkTestCase, ReusedPySparkTestCase, SQLTestUtils): +class SparkConnectFuncTestCase(PandasOnSparkTestCase, ReusedPySparkTestCase, SQLTestUtils): """Parent test fixture class for all Spark Connect related test cases.""" @@ -50,50 +49,15 @@ class SparkConnectSQLTestCase(PandasOnSparkTestCase, ReusedPySparkTestCase, SQLT cls.hive_available = True # Create the new Spark Session cls.spark = SparkSession(cls.sc) - cls.testData = [Row(key=i, value=str(i)) for i in range(100)] - cls.testDataStr = [Row(key=str(i)) for i in range(100)] - cls.df = cls.sc.parallelize(cls.testData).toDF() - cls.df_text = cls.sc.parallelize(cls.testDataStr).toDF() - - cls.tbl_name = "test_connect_basic_table_1" - cls.tbl_name_empty = "test_connect_basic_table_empty" - - # Cleanup test data - cls.spark_connect_clean_up_test_data() - # Load test data - cls.spark_connect_load_test_data() - - @classmethod - def tearDownClass(cls: Any) -> None: - cls.spark_connect_clean_up_test_data() - ReusedPySparkTestCase.tearDownClass() - - @classmethod - def spark_connect_load_test_data(cls: Any): # Setup Remote Spark Session cls.connect = RemoteSparkSession.builder.remote().getOrCreate() - df = cls.spark.createDataFrame([(x, f"{x}") for x in range(100)], ["id", "name"]) - # Since we might create multiple Spark sessions, we need to create global temporary view - # that is specifically maintained in the "global_temp" schema. - df.write.saveAsTable(cls.tbl_name) - empty_table_schema = StructType( - [ - StructField("firstname", StringType(), True), - StructField("middlename", StringType(), True), - StructField("lastname", StringType(), True), - ] - ) - emptyRDD = cls.spark.sparkContext.emptyRDD() - empty_df = cls.spark.createDataFrame(emptyRDD, empty_table_schema) - empty_df.write.saveAsTable(cls.tbl_name_empty) @classmethod - def spark_connect_clean_up_test_data(cls: Any) -> None: - cls.spark.sql("DROP TABLE IF EXISTS {}".format(cls.tbl_name)) - cls.spark.sql("DROP TABLE IF EXISTS {}".format(cls.tbl_name_empty)) + def tearDownClass(cls: Any) -> None: + ReusedPySparkTestCase.tearDownClass() -class SparkConnectFunctionTests(SparkConnectSQLTestCase): +class SparkConnectFunctionTests(SparkConnectFuncTestCase): """These test cases exercise the interface to the proto plan generation but do not call Spark.""" --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org