This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 4aeddb8 [SPARK-36870][SQL] Introduce INTERNAL_ERROR error class
4aeddb8 is described below
commit 4aeddb81d1cc9c464d5dddbd4d54dda4836a6078
Author: Karen Feng <[email protected]>
AuthorDate: Fri Oct 1 13:46:45 2021 +0900
[SPARK-36870][SQL] Introduce INTERNAL_ERROR error class
### What changes were proposed in this pull request?
Adds the `INTERNAL_ERROR` error class and the `isInternalError` API to
`SparkThrowable`.
Removes existing error classes that are internal-only and replaces them
with `INTERNAL_ERROR`.
### Why are the changes needed?
Makes it easy for end-users to diagnose whether an error is an internal
error. If an end-user encounters an internal error, it should be reported
immediately. This also limits the number of error classes, making it easy to
audit. We do not need high-quality error messages for internal errors, as they
should not be exposed to the end-user.
### Does this PR introduce _any_ user-facing change?
Yes; this changes the error class in master.
### How was this patch tested?
Unit tests
Closes #34123 from karenfeng/internal-error-class.
Authored-by: Karen Feng <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../main/java/org/apache/spark/SparkThrowable.java | 9 +++++++-
.../apache/spark/memory/SparkOutOfMemoryError.java | 4 ----
core/src/main/resources/error/README.md | 19 ++++++++--------
core/src/main/resources/error/error-classes.json | 19 +++-------------
.../main/scala/org/apache/spark/ErrorInfo.scala | 4 ++++
.../scala/org/apache/spark/SparkException.scala | 20 -----------------
.../org/apache/spark/SparkThrowableSuite.scala | 17 +++++++++++++++
.../org/apache/spark/sql/AnalysisException.scala | 1 -
.../spark/sql/errors/QueryExecutionErrors.scala | 25 ++++++++++++----------
9 files changed, 56 insertions(+), 62 deletions(-)
diff --git a/core/src/main/java/org/apache/spark/SparkThrowable.java
b/core/src/main/java/org/apache/spark/SparkThrowable.java
index 31a9ab0..2be0c3c 100644
--- a/core/src/main/java/org/apache/spark/SparkThrowable.java
+++ b/core/src/main/java/org/apache/spark/SparkThrowable.java
@@ -38,5 +38,12 @@ public interface SparkThrowable {
// Portable error identifier across SQL engines
// If null, error class or SQLSTATE is not set
- String getSqlState();
+ default String getSqlState() {
+ return SparkThrowableHelper.getSqlState(this.getErrorClass());
+ }
+
+ // True if this error is an internal error.
+ default boolean isInternalError() {
+ return SparkThrowableHelper.isInternalError(this.getErrorClass());
+ }
}
diff --git
a/core/src/main/java/org/apache/spark/memory/SparkOutOfMemoryError.java
b/core/src/main/java/org/apache/spark/memory/SparkOutOfMemoryError.java
index bf7984b..88eada3 100644
--- a/core/src/main/java/org/apache/spark/memory/SparkOutOfMemoryError.java
+++ b/core/src/main/java/org/apache/spark/memory/SparkOutOfMemoryError.java
@@ -47,8 +47,4 @@ public final class SparkOutOfMemoryError extends
OutOfMemoryError implements Spa
public String getErrorClass() {
return errorClass;
}
-
- public String getSqlState() {
- return SparkThrowableHelper.getSqlState(errorClass);
- }
}
diff --git a/core/src/main/resources/error/README.md
b/core/src/main/resources/error/README.md
index c4ab6bb..f58eb6d 100644
--- a/core/src/main/resources/error/README.md
+++ b/core/src/main/resources/error/README.md
@@ -5,13 +5,16 @@ and message parameters rather than an arbitrary error message.
## Usage
-1. Check if an appropriate error class already exists in `error-class.json`.
- If true, skip to step 3. Otherwise, continue to step 2.
-2. Add a new class to `error-class.json`; keep in mind the invariants below.
-3. Check if the exception type already extends `SparkThrowable`.
- If true, skip to step 5. Otherwise, continue to step 4.
-4. Mix `SparkThrowable` into the exception.
-5. Throw the exception with the error class and message parameters.
+1. Check if the error is an internal error.
+ Internal errors are bugs in the code that we do not expect users to
encounter; this does not include unsupported operations.
+ If true, use the error class `INTERNAL_ERROR` and skip to step 4.
+2. Check if an appropriate error class already exists in `error-class.json`.
+ If true, use the error class and skip to step 4.
+3. Add a new class to `error-class.json`; keep in mind the invariants below.
+4. Check if the exception type already extends `SparkThrowable`.
+ If true, skip to step 6.
+5. Mix `SparkThrowable` into the exception.
+6. Throw the exception with the error class and message parameters.
### Before
@@ -37,8 +40,6 @@ Throw with arbitrary error message:
with SparkThrowable {
def getErrorClass: String = errorClass
- def getMessageParameters: Array[String] = messageParameters
- def getSqlState: String = SparkThrowableHelper.getSqlState(errorClass)
}
Throw with error class and message parameters:
diff --git a/core/src/main/resources/error/error-classes.json
b/core/src/main/resources/error/error-classes.json
index 31b2144..d270f0e 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -11,19 +11,10 @@
"message" : [ "%s cannot be represented as Decimal(%s, %s)." ],
"sqlState" : "22005"
},
- "CANNOT_EVALUATE_EXPRESSION" : {
- "message" : [ "Cannot evaluate expression: %s" ]
- },
- "CANNOT_GENERATE_CODE_FOR_EXPRESSION" : {
- "message" : [ "Cannot generate code for expression: %s" ]
- },
"CANNOT_PARSE_DECIMAL" : {
"message" : [ "Cannot parse decimal" ],
"sqlState" : "42000"
},
- "CANNOT_TERMINATE_GENERATOR" : {
- "message" : [ "Cannot terminate expression: %s" ]
- },
"CAST_CAUSES_OVERFLOW" : {
"message" : [ "Casting %s to %s causes overflow" ],
"sqlState" : "22005"
@@ -74,6 +65,9 @@
"message" : [ "Index %s must be between 0 and the length of the
ArrayData." ],
"sqlState" : "22023"
},
+ "INTERNAL_ERROR" : {
+ "message" : [ "%s" ]
+ },
"INVALID_ARRAY_INDEX" : {
"message" : [ "Invalid index: %s, numElements: %s" ]
},
@@ -92,9 +86,6 @@
"INVALID_JSON_SCHEMA_MAPTYPE" : {
"message" : [ "Input schema %s can only contain StringType as a key type
for a MapType." ]
},
- "LOGICAL_HINT_OPERATOR_NOT_REMOVED_DURING_ANALYSIS" : {
- "message" : [ "Internal error: logical hint operator should have been
removed during analysis" ]
- },
"MAP_KEY_DOES_NOT_EXIST" : {
"message" : [ "Key %s does not exist." ]
},
@@ -102,10 +93,6 @@
"message" : [ "cannot resolve '%s' given input columns: [%s]" ],
"sqlState" : "42000"
},
- "MISSING_METHOD" : {
- "message" : [ "A method named \"%s\" is not declared in any enclosing
class nor any supertype" ],
- "sqlState" : "42000"
- },
"MISSING_STATIC_PARTITION_COLUMN" : {
"message" : [ "Unknown static partition column: %s" ],
"sqlState" : "42000"
diff --git a/core/src/main/scala/org/apache/spark/ErrorInfo.scala
b/core/src/main/scala/org/apache/spark/ErrorInfo.scala
index 67a363c..0917085 100644
--- a/core/src/main/scala/org/apache/spark/ErrorInfo.scala
+++ b/core/src/main/scala/org/apache/spark/ErrorInfo.scala
@@ -64,4 +64,8 @@ private[spark] object SparkThrowableHelper {
def getSqlState(errorClass: String): String = {
Option(errorClass).flatMap(errorClassToInfoMap.get).flatMap(_.sqlState).orNull
}
+
+ def isInternalError(errorClass: String): Boolean = {
+ errorClass == "INTERNAL_ERROR"
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/SparkException.scala
b/core/src/main/scala/org/apache/spark/SparkException.scala
index c508864..aea09e3 100644
--- a/core/src/main/scala/org/apache/spark/SparkException.scala
+++ b/core/src/main/scala/org/apache/spark/SparkException.scala
@@ -45,7 +45,6 @@ class SparkException(
messageParameters = messageParameters)
override def getErrorClass: String = errorClass.orNull
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass.orNull)
}
/**
@@ -84,7 +83,6 @@ private[spark] class SparkArithmeticException(errorClass:
String, messageParamet
with SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -97,7 +95,6 @@ private[spark] class SparkUnsupportedOperationException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -111,7 +108,6 @@ private[spark] class SparkClassNotFoundException(
SparkThrowableHelper.getMessage(errorClass, messageParameters), cause)
with SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -125,7 +121,6 @@ private[spark] class SparkConcurrentModificationException(
SparkThrowableHelper.getMessage(errorClass, messageParameters), cause)
with SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -136,7 +131,6 @@ private[spark] class SparkDateTimeException(errorClass:
String, messageParameter
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -149,7 +143,6 @@ private[spark] class SparkFileAlreadyExistsException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -162,7 +155,6 @@ private[spark] class SparkIllegalStateException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -175,7 +167,6 @@ private[spark] class SparkFileNotFoundException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -188,7 +179,6 @@ private[spark] class SparkNumberFormatException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -201,7 +191,6 @@ private[spark] class SparkNoSuchMethodException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -214,7 +203,6 @@ private[spark] class SparkIllegalArgumentException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -227,7 +215,6 @@ private[spark] class SparkIndexOutOfBoundsException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -240,7 +227,6 @@ private[spark] class SparkIOException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
private[spark] class SparkRuntimeException(
@@ -251,7 +237,6 @@ private[spark] class SparkRuntimeException(
SparkThrowableHelper.getMessage(errorClass, messageParameters), cause)
with SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -264,7 +249,6 @@ private[spark] class SparkSecurityException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -277,7 +261,6 @@ private[spark] class SparkArrayIndexOutOfBoundsException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -290,7 +273,6 @@ private[spark] class SparkSQLException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -303,7 +285,6 @@ private[spark] class SparkNoSuchElementException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
/**
@@ -316,5 +297,4 @@ private[spark] class SparkSQLFeatureNotSupportedException(
SparkThrowableHelper.getMessage(errorClass, messageParameters)) with
SparkThrowable {
override def getErrorClass: String = errorClass
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass)
}
diff --git a/core/src/test/scala/org/apache/spark/SparkThrowableSuite.scala
b/core/src/test/scala/org/apache/spark/SparkThrowableSuite.scala
index 750b2ee..5af55af 100644
--- a/core/src/test/scala/org/apache/spark/SparkThrowableSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkThrowableSuite.scala
@@ -160,4 +160,21 @@ class SparkThrowableSuite extends SparkFunSuite {
assert(false)
}
}
+
+ test("Try catching internal SparkError") {
+ try {
+ throw new SparkException(
+ errorClass = "INTERNAL_ERROR",
+ messageParameters = Array("this is an internal error"),
+ cause = null
+ )
+ } catch {
+ case e: SparkThrowable =>
+ assert(e.isInternalError)
+ assert(e.getSqlState == null)
+ case _: Throwable =>
+ // Should not end up here
+ assert(false)
+ }
+ }
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
index 35eb040..397300d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
@@ -92,5 +92,4 @@ class AnalysisException protected[sql] (
}
override def getErrorClass: String = errorClass.orNull
- override def getSqlState: String =
SparkThrowableHelper.getSqlState(errorClass.orNull)
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index 8e43f48..3fe367b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -72,23 +72,24 @@ object QueryExecutionErrors {
}
def logicalHintOperatorNotRemovedDuringAnalysisError(): Throwable = {
- new SparkIllegalStateException(errorClass =
"LOGICAL_HINT_OPERATOR_NOT_REMOVED_DURING_ANALYSIS",
- messageParameters = Array.empty)
+ new SparkIllegalStateException(errorClass = "INTERNAL_ERROR",
+ messageParameters = Array(
+ "Internal error: logical hint operator should have been removed during
analysis"))
}
def cannotEvaluateExpressionError(expression: Expression): Throwable = {
- new SparkUnsupportedOperationException(errorClass =
"CANNOT_EVALUATE_EXPRESSION",
- messageParameters = Array("", expression.toString))
+ new SparkUnsupportedOperationException(errorClass = "INTERNAL_ERROR",
+ messageParameters = Array(s"Cannot evaluate expression: $expression"))
}
def cannotGenerateCodeForExpressionError(expression: Expression): Throwable
= {
- new SparkUnsupportedOperationException(errorClass =
"CANNOT_GENERATE_CODE_FOR_EXPRESSION",
- messageParameters = Array(expression.toString))
+ new SparkUnsupportedOperationException(errorClass = "INTERNAL_ERROR",
+ messageParameters = Array(s"Cannot generate code for expression:
$expression"))
}
def cannotTerminateGeneratorError(generator: UnresolvedGenerator): Throwable
= {
- new SparkUnsupportedOperationException(errorClass =
"CANNOT_TERMINATE_GENERATOR",
- messageParameters = Array(generator.toString))
+ new SparkUnsupportedOperationException(errorClass = "INTERNAL_ERROR",
+ messageParameters = Array(s"Cannot terminate expression: $generator"))
}
def castingCauseOverflowError(t: Any, targetType: String):
ArithmeticException = {
@@ -130,8 +131,8 @@ object QueryExecutionErrors {
def evaluateUnevaluableAggregateUnsupportedError(
methodName: String, unEvaluable: UnevaluableAggregate): Throwable = {
- new SparkUnsupportedOperationException(errorClass =
"CANNOT_EVALUATE_EXPRESSION",
- messageParameters = Array(methodName + ": " + unEvaluable.toString))
+ new SparkUnsupportedOperationException(errorClass = "INTERNAL_ERROR",
+ messageParameters = Array(s"Cannot evaluate expression: $methodName:
$unEvaluable"))
}
def dataTypeUnsupportedError(dt: DataType): Throwable = {
@@ -279,7 +280,9 @@ object QueryExecutionErrors {
}
def methodNotDeclaredError(name: String): Throwable = {
- new SparkNoSuchMethodException(errorClass = "MISSING_METHOD", Array(name))
+ new SparkNoSuchMethodException(errorClass = "INTERNAL_ERROR",
+ messageParameters = Array(
+ s"""A method named "$name" is not declared in any enclosing class nor
any supertype"""))
}
def constructorNotFoundError(cls: String): Throwable = {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
