This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 542644075b3 [SPARK-41892][CONNECT][TESTS]
pyspark.sql.tests.test_functions - Add JIRAs or messages for skipped messages
542644075b3 is described below
commit 542644075b3fe92ca2b6675111237fd2fc177ba1
Author: Sandeep Singh <[email protected]>
AuthorDate: Fri Jan 6 09:42:44 2023 +0900
[SPARK-41892][CONNECT][TESTS] pyspark.sql.tests.test_functions - Add JIRAs
or messages for skipped messages
### What changes were proposed in this pull request?
This PR enables the reused PySpark tests in Spark Connect that pass now.
And add JIRAs/ Messages to the skipped ones
### Why are the changes needed?
To make sure on the test coverage.
### Does this PR introduce any user-facing change?
No, test-only.
### How was this patch tested?
Enabling tests
Closes #39412 from techaddict/SPARK-41892.
Authored-by: Sandeep Singh <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../sql/tests/connect/test_parity_functions.py | 26 ++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/python/pyspark/sql/tests/connect/test_parity_functions.py
b/python/pyspark/sql/tests/connect/test_parity_functions.py
index 3c616b5c864..a85acf7f6ec 100644
--- a/python/pyspark/sql/tests/connect/test_parity_functions.py
+++ b/python/pyspark/sql/tests/connect/test_parity_functions.py
@@ -44,106 +44,132 @@ class FunctionsParityTests(ReusedSQLTestCase,
FunctionsTestsMixin):
cls.spark = cls._spark.stop()
del os.environ["SPARK_REMOTE"]
+ # TODO(SPARK-41897): Parity in Error types between pyspark and connect
functions
@unittest.skip("Fails in Spark Connect, should enable.")
def test_assert_true(self):
super().test_assert_true()
+ # Spark Connect does not support Spark Context but the test depends on
that.
@unittest.skip("Fails in Spark Connect, should enable.")
def test_basic_functions(self):
super().test_basic_functions()
+ # TODO(SPARK-41899): DataFrame.createDataFrame converting int to bigint
@unittest.skip("Fails in Spark Connect, should enable.")
def test_date_add_function(self):
super().test_date_add_function()
+ # TODO(SPARK-41899): DataFrame.createDataFrame converting int to bigint
@unittest.skip("Fails in Spark Connect, should enable.")
def test_date_sub_function(self):
super().test_date_sub_function()
+ # TODO(SPARK-41847): DataFrame mapfield,structlist invalid type
@unittest.skip("Fails in Spark Connect, should enable.")
def test_explode(self):
super().test_explode()
+ # Spark Connect does not support Spark Context but the test depends on
that.
@unittest.skip("Fails in Spark Connect, should enable.")
def test_function_parity(self):
super().test_function_parity()
+ # Spark Connect does not support Spark Context, _jdf but the test depends
on that.
@unittest.skip("Fails in Spark Connect, should enable.")
def test_functions_broadcast(self):
super().test_functions_broadcast()
+ # Spark Connect does not support Spark Context but the test depends on
that.
@unittest.skip("Fails in Spark Connect, should enable.")
def test_input_file_name_reset_for_rdd(self):
super().test_input_file_name_reset_for_rdd()
+ # TODO(SPARK-41849): Implement DataFrameReader.text
@unittest.skip("Fails in Spark Connect, should enable.")
def test_input_file_name_udf(self):
super().test_input_file_name_udf()
+ # TODO(SPARK-41901): Parity in String representation of Column
@unittest.skip("Fails in Spark Connect, should enable.")
def test_inverse_trig_functions(self):
super().test_inverse_trig_functions()
+ # TODO(SPARK-41834): Implement SparkSession.conf
@unittest.skip("Fails in Spark Connect, should enable.")
def test_lit_list(self):
super().test_lit_list()
+ # TODO(SPARK-41900): support Data Type int8
@unittest.skip("Fails in Spark Connect, should enable.")
def test_lit_np_scalar(self):
super().test_lit_np_scalar()
+ # TODO(SPARK-41902): Fix String representation of maps created by
`map_from_arrays`
@unittest.skip("Fails in Spark Connect, should enable.")
def test_map_functions(self):
super().test_map_functions()
+ # TODO(SPARK-41903): Support data type ndarray
@unittest.skip("Fails in Spark Connect, should enable.")
def test_ndarray_input(self):
super().test_ndarray_input()
+ # TODO(SPARK-41902): Parity in String representation of
higher_order_function's output
@unittest.skip("Fails in Spark Connect, should enable.")
def test_nested_higher_order_function(self):
super().test_nested_higher_order_function()
+ # TODO(SPARK-41900): support Data Type int8
@unittest.skip("Fails in Spark Connect, should enable.")
def test_np_scalar_input(self):
super().test_np_scalar_input()
+ # TODO(SPARK-41904): Fix nth_value value
@unittest.skip("Fails in Spark Connect, should enable.")
def test_nth_value(self):
super().test_nth_value()
+ # TODO(SPARK-41901): Parity in String representation of Column
@unittest.skip("Fails in Spark Connect, should enable.")
def test_overlay(self):
super().test_overlay()
+ # TODO(SPARK-41901): Parity in String representation of Column
@unittest.skip("Fails in Spark Connect, should enable.")
def test_percentile_approx(self):
super().test_percentile_approx()
+ # TODO(SPARK-41897): Parity in Error types between pyspark and connect
functions
@unittest.skip("Fails in Spark Connect, should enable.")
def test_raise_error(self):
super().test_raise_error()
+ # TODO(SPARK-41905): Function `slice` should expect string
@unittest.skip("Fails in Spark Connect, should enable.")
def test_slice(self):
super().test_slice()
+ # Comparing column type of connect and pyspark
@unittest.skip("Fails in Spark Connect, should enable.")
def test_sorting_functions_with_column(self):
super().test_sorting_functions_with_column()
+ # TODO(SPARK-41898): support float("-inf") Window.rowsBetween
@unittest.skip("Fails in Spark Connect, should enable.")
def test_window_functions(self):
super().test_window_functions()
+ # TODO(SPARK-41898): support float("-inf") Window.rowsBetween
@unittest.skip("Fails in Spark Connect, should enable.")
def test_window_functions_without_partitionBy(self):
super().test_window_functions_without_partitionBy()
+ # TODO(SPARK-41906): Handle Function `rand() `
@unittest.skip("Fails in Spark Connect, should enable.")
def test_rand_functions(self):
super().test_rand_functions()
+ # TODO(SPARK-41907): sampleby returning wrong output
@unittest.skip("Fails in Spark Connect, should enable.")
def test_sampleby(self):
super().test_sampleby()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
