This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6120c5682af [SPARK-40448][CONNECT][FOLLOWUP] Remove
`InputValidationError` and turn into error class
6120c5682af is described below
commit 6120c5682afc507b709261648b07f3a6555f5eff
Author: itholic <[email protected]>
AuthorDate: Fri May 5 10:36:05 2023 +0800
[SPARK-40448][CONNECT][FOLLOWUP] Remove `InputValidationError` and turn
into error class
### What changes were proposed in this pull request?
This PR proposes to remove `InputValidationError` and use
`PySparkTypeError` instead.
### Why are the changes needed?
We should use PySpark specific errors for every errors raised from PySpark.
### Does this PR introduce _any_ user-facing change?
No, it's error improvement.
### How was this patch tested?
The existing CI should pass.
Closes #40987 from itholic/error_inputval_followup.
Authored-by: itholic <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/errors/error_classes.py | 5 +++++
python/pyspark/sql/connect/plan.py | 11 ++++-------
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/python/pyspark/errors/error_classes.py
b/python/pyspark/errors/error_classes.py
index c2ed64d01fb..c6f335bf8ac 100644
--- a/python/pyspark/errors/error_classes.py
+++ b/python/pyspark/errors/error_classes.py
@@ -425,6 +425,11 @@ ERROR_CLASSES_JSON = """
"Argument `<arg_name>` should be a list[Column]."
]
},
+ "NOT_LIST_OF_COLUMN_OR_STR" : {
+ "message" : [
+ "Argument `<arg_name>` should be a list[Column]."
+ ]
+ },
"NOT_LIST_OF_FLOAT_OR_INT" : {
"message" : [
"Argument `<arg_name>` should be a list[float, int], got <arg_type>."
diff --git a/python/pyspark/sql/connect/plan.py
b/python/pyspark/sql/connect/plan.py
index f7000732acd..03aca4896be 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -38,7 +38,7 @@ from pyspark.sql.connect.expressions import (
LiteralExpression,
)
from pyspark.sql.connect.types import pyspark_types_to_proto_types
-from pyspark.errors import PySparkNotImplementedError
+from pyspark.errors import PySparkTypeError, PySparkNotImplementedError
if TYPE_CHECKING:
from pyspark.sql.connect._typing import ColumnOrName
@@ -46,10 +46,6 @@ if TYPE_CHECKING:
from pyspark.sql.connect.udf import UserDefinedFunction
-class InputValidationError(Exception):
- pass
-
-
class LogicalPlan:
_lock: Lock = Lock()
@@ -433,8 +429,9 @@ class Project(LogicalPlan):
"""Ensures that all input arguments are instances of Expression or
String."""
for c in self._columns:
if not isinstance(c, (Column, str)):
- raise InputValidationError(
- f"Only Column or String can be used for projections:
'{c}'."
+ raise PySparkTypeError(
+ error_class="NOT_LIST_OF_COLUMN_OR_STR",
+ message_parameters={"arg_name": "columns"},
)
def plan(self, session: "SparkConnectClient") -> proto.Relation:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
