This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 78aa3e04558e [SPARK-58214][PYTHON] Consolidate VALUE_NOT_PEARSON into
VALUE_NOT_ALLOWED
78aa3e04558e is described below
commit 78aa3e04558e6c8b5f2594f136a9d45e84c1c4f8
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Jul 21 08:33:07 2026 +0800
[SPARK-58214][PYTHON] Consolidate VALUE_NOT_PEARSON into VALUE_NOT_ALLOWED
### What changes were proposed in this pull request?
Consolidate the specialized PySpark `VALUE_NOT_PEARSON` error condition
into `VALUE_NOT_ALLOWED`. Classic and Spark Connect `DataFrame.stat.corr` now
report the generic condition when `method` is not `pearson`. The Spark Connect
statistics test asserts the new condition.
### Why are the changes needed?
`VALUE_NOT_PEARSON` only represents a single allowed argument value and
duplicates the generic allowed-values condition. Removing it reduces narrowly
scoped error conditions and aligns the validation with other PySpark APIs.
### Does this PR introduce _any_ user-facing change?
Yes. Invalid `DataFrame.stat.corr(..., method=...)` calls now use
`VALUE_NOT_ALLOWED` and its generic message. The exception type remains
`PySparkValueError`.
### How was this patch tested?
Updated the Spark Connect statistics test assertion. The focused PySpark
test suite was not run.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57366 from zhengruifeng/value-not-allowed-dev1.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/errors/error-conditions.json | 5 -----
python/pyspark/sql/classic/dataframe.py | 4 ++--
python/pyspark/sql/connect/dataframe.py | 4 ++--
python/pyspark/sql/tests/connect/test_connect_stat.py | 15 ++++++++-------
4 files changed, 12 insertions(+), 16 deletions(-)
diff --git a/python/pyspark/errors/error-conditions.json
b/python/pyspark/errors/error-conditions.json
index 6c7612da286d..07f47a0c9100 100644
--- a/python/pyspark/errors/error-conditions.json
+++ b/python/pyspark/errors/error-conditions.json
@@ -1208,11 +1208,6 @@
"Value for `<arg_name>` must be a non-empty string, got '<arg_value>'."
]
},
- "VALUE_NOT_PEARSON": {
- "message": [
- "Value for `<arg_name>` only supports 'pearson', got '<arg_value>'."
- ]
- },
"VALUE_NOT_PLAIN_COLUMN_REFERENCE": {
"message": [
"Value `<val>` in `<field_name>` should be a plain column reference such
as `df.col` or `col('column')`."
diff --git a/python/pyspark/sql/classic/dataframe.py
b/python/pyspark/sql/classic/dataframe.py
index ca4b1f17de2a..9599fad3bcda 100644
--- a/python/pyspark/sql/classic/dataframe.py
+++ b/python/pyspark/sql/classic/dataframe.py
@@ -1693,8 +1693,8 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin,
PandasConversionMixin):
method = "pearson"
if not method == "pearson":
raise PySparkValueError(
- errorClass="VALUE_NOT_PEARSON",
- messageParameters={"arg_name": "method", "arg_value": method},
+ errorClass="VALUE_NOT_ALLOWED",
+ messageParameters={"arg_name": "method", "allowed_values":
"['pearson']"},
)
return self._jdf.stat().corr(col1, col2, method)
diff --git a/python/pyspark/sql/connect/dataframe.py
b/python/pyspark/sql/connect/dataframe.py
index 64277d1b35b6..99ee07709937 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -1663,8 +1663,8 @@ class DataFrame(ParentDataFrame):
method = "pearson"
if not method == "pearson":
raise PySparkValueError(
- errorClass="VALUE_NOT_PEARSON",
- messageParameters={"arg_name": "method", "arg_value": method},
+ errorClass="VALUE_NOT_ALLOWED",
+ messageParameters={"arg_name": "method", "allowed_values":
"['pearson']"},
)
table, _ = DataFrame(
plan.StatCorr(child=self._plan, col1=col1, col2=col2,
method=method),
diff --git a/python/pyspark/sql/tests/connect/test_connect_stat.py
b/python/pyspark/sql/tests/connect/test_connect_stat.py
index c5f686899637..3d05cb7b4bb5 100644
--- a/python/pyspark/sql/tests/connect/test_connect_stat.py
+++ b/python/pyspark/sql/tests/connect/test_connect_stat.py
@@ -214,13 +214,14 @@ class SparkConnectStatTests(SparkConnectSQLTestCase):
"arg_type": "int",
},
)
- with self.assertRaises(ValueError) as context:
- (self.connect.read.table(self.tbl_name2).stat.corr("col1", "col3",
"spearman"),)
- self.assertTrue(
- "Currently only the calculation of the Pearson Correlation "
- + "coefficient is supported."
- in str(context.exception)
- )
+ with self.assertRaises(PySparkValueError) as pe:
+ self.connect.read.table(self.tbl_name2).stat.corr("col1", "col3",
"spearman")
+
+ self.check_error(
+ exception=pe.exception,
+ errorClass="VALUE_NOT_ALLOWED",
+ messageParameters={"arg_name": "method", "allowed_values":
"['pearson']"},
+ )
def test_stat_approx_quantile(self):
# SPARK-41069: Test the stat.approxQuantile method
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
