HyukjinKwon commented on code in PR #53188:
URL: https://github.com/apache/spark/pull/53188#discussion_r2554433303
##########
python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py:
##########
@@ -243,6 +244,7 @@ def merge_pandas(lft, rgt):
self._test_merge_empty(fn=merge_pandas)
+ @unittest.skipIf(os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1",
"To be reenabled")
Review Comment:
```
======================================================================
FAIL [0.000s]: test_apply_in_pandas_returning_incompatible_type
(pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map.CogroupedApplyInPandasTests.test_apply_in_pandas_returning_incompatible_type)
[without key] (convert='string to double', convertToArrowArraySafely=True)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 574, in _test_merge_error
self.__test_merge_error(
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 606, in __test_merge_error
self.__test_merge(left, right, by, fn, output_schema)
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 557, in __test_merge
assert_frame_equal(expected, result)
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 1218, in assert_frame_equal
raise_assert_detail(
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 614, in raise_assert_detail
raise AssertionError(msg)
AssertionError: DataFrame are different
DataFrame shape mismatch
[left]: (100, 4)
[right]: (10, 2)
======================================================================
FAIL [0.000s]: test_apply_in_pandas_returning_incompatible_type
(pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map.CogroupedApplyInPandasTests.test_apply_in_pandas_returning_incompatible_type)
[with key] (convert='string to double', convertToArrowArraySafely=True)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 584, in _test_merge_error
self.__test_merge_error(
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 606, in __test_merge_error
self.__test_merge(left, right, by, fn, output_schema)
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 557, in __test_merge
assert_frame_equal(expected, result)
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 1218, in assert_frame_equal
raise_assert_detail(
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 614, in raise_assert_detail
raise AssertionError(msg)
AssertionError: DataFrame are different
DataFrame shape mismatch
[left]: (100, 4)
[right]: (10, 2)
======================================================================
FAIL [0.000s]: test_apply_in_pandas_returning_incompatible_type
(pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map.CogroupedApplyInPandasTests.test_apply_in_pandas_returning_incompatible_type)
[without key] (convert='string to double', convertToArrowArraySafely=False)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 574, in _test_merge_error
self.__test_merge_error(
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 606, in __test_merge_error
self.__test_merge(left, right, by, fn, output_schema)
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 557, in __test_merge
assert_frame_equal(expected, result)
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 1218, in assert_frame_equal
raise_assert_detail(
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 614, in raise_assert_detail
raise AssertionError(msg)
AssertionError: DataFrame are different
DataFrame shape mismatch
[left]: (100, 4)
[right]: (10, 2)
======================================================================
FAIL [3.160s]: test_apply_in_pandas_returning_incompatible_type
(pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map.CogroupedApplyInPandasTests.test_apply_in_pandas_returning_incompatible_type)
[with key] (convert='string to double', convertToArrowArraySafely=False)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 584, in _test_merge_error
self.__test_merge_error(
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 606, in __test_merge_error
self.__test_merge(left, right, by, fn, output_schema)
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 557, in __test_merge
assert_frame_equal(expected, result)
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 1218, in assert_frame_equal
raise_assert_detail(
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 614, in raise_assert_detail
raise AssertionError(msg)
AssertionError: DataFrame are different
DataFrame shape mismatch
[left]: (100, 4)
[right]: (10, 2)
----------------------------------------------------------------------
```
##########
python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py:
##########
@@ -335,6 +336,7 @@ def
check_apply_in_pandas_returning_no_column_names_and_wrong_amount(self):
def test_apply_in_pandas_returning_empty_dataframe(self):
self._test_apply_in_pandas_returning_empty_dataframe(pd.DataFrame())
+ @unittest.skipIf(os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1",
"To be reenabled")
Review Comment:
```
======================================================================
FAIL [0.000s]: test_apply_in_pandas_returning_incompatible_type
(pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map.CogroupedApplyInPandasTests.test_apply_in_pandas_returning_incompatible_type)
[without key] (convert='string to double', convertToArrowArraySafely=True)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 574, in _test_merge_error
self.__test_merge_error(
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 606, in __test_merge_error
self.__test_merge(left, right, by, fn, output_schema)
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 557, in __test_merge
assert_frame_equal(expected, result)
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 1218, in assert_frame_equal
raise_assert_detail(
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 614, in raise_assert_detail
raise AssertionError(msg)
AssertionError: DataFrame are different
DataFrame shape mismatch
[left]: (100, 4)
[right]: (10, 2)
======================================================================
FAIL [0.000s]: test_apply_in_pandas_returning_incompatible_type
(pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map.CogroupedApplyInPandasTests.test_apply_in_pandas_returning_incompatible_type)
[with key] (convert='string to double', convertToArrowArraySafely=True)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 584, in _test_merge_error
self.__test_merge_error(
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 606, in __test_merge_error
self.__test_merge(left, right, by, fn, output_schema)
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 557, in __test_merge
assert_frame_equal(expected, result)
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 1218, in assert_frame_equal
raise_assert_detail(
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 614, in raise_assert_detail
raise AssertionError(msg)
AssertionError: DataFrame are different
DataFrame shape mismatch
[left]: (100, 4)
[right]: (10, 2)
======================================================================
FAIL [0.000s]: test_apply_in_pandas_returning_incompatible_type
(pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map.CogroupedApplyInPandasTests.test_apply_in_pandas_returning_incompatible_type)
[without key] (convert='string to double', convertToArrowArraySafely=False)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 574, in _test_merge_error
self.__test_merge_error(
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 606, in __test_merge_error
self.__test_merge(left, right, by, fn, output_schema)
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 557, in __test_merge
assert_frame_equal(expected, result)
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 1218, in assert_frame_equal
raise_assert_detail(
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 614, in raise_assert_detail
raise AssertionError(msg)
AssertionError: DataFrame are different
DataFrame shape mismatch
[left]: (100, 4)
[right]: (10, 2)
======================================================================
FAIL [3.160s]: test_apply_in_pandas_returning_incompatible_type
(pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map.CogroupedApplyInPandasTests.test_apply_in_pandas_returning_incompatible_type)
[with key] (convert='string to double', convertToArrowArraySafely=False)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 584, in _test_merge_error
self.__test_merge_error(
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 606, in __test_merge_error
self.__test_merge(left, right, by, fn, output_schema)
File
"/home/runner/work/spark/spark-4.0/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py",
line 557, in __test_merge
assert_frame_equal(expected, result)
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 1218, in assert_frame_equal
raise_assert_detail(
File
"/opt/hostedtoolcache/Python/3.11.14/x64/lib/python3.11/site-packages/pandas/_testing/asserters.py",
line 614, in raise_assert_detail
raise AssertionError(msg)
AssertionError: DataFrame are different
DataFrame shape mismatch
[left]: (100, 4)
[right]: (10, 2)
----------------------------------------------------------------------
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
