MaxGekk commented on code in PR #39937:
URL: https://github.com/apache/spark/pull/39937#discussion_r1120255417
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala:
##########
@@ -496,12 +497,13 @@ object DataType {
case (wArr: ArrayType, rArr: ArrayType) =>
// run compatibility check first to produce all error messages
val typesCompatible = canWrite(
- wArr.elementType, rArr.elementType, byName, resolver, context +
".element",
+ tableName, wArr.elementType, rArr.elementType, byName, resolver,
context + ".element",
storeAssignmentPolicy, addError)
if (wArr.containsNull && !rArr.containsNull) {
- addError(s"Cannot write nullable elements to array of non-nulls:
'$context'")
Review Comment:
Is the param `addError` still used somewhere?
##########
docs/sql-ref-ansi-compliance.md:
##########
@@ -151,7 +151,7 @@ CREATE TABLE t (v INT);
-- `spark.sql.storeAssignmentPolicy=ANSI`
INSERT INTO t VALUES ('1');
-org.apache.spark.sql.AnalysisException: Cannot write incompatible data to
table '`default`.`t`':
+org.apache.spark.sql.AnalysisException: Cannot write incompatible data for
table '`default`.`t`':
- Cannot safely cast 'v': string to int;
Review Comment:
The message should look differently after your changes. Could you re-run the
example, and change the output.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala:
##########
@@ -485,6 +485,7 @@ object DataType {
* @return true if data written with the write type can be read using the
read type
*/
def canWrite(
+ tableName: String,
Review Comment:
Please, update mima rules.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala:
##########
@@ -83,11 +82,13 @@ object TableOutputResolver {
val matched = inputCols.filter(col => conf.resolver(col.name,
expectedCol.name))
val newColPath = colPath :+ expectedCol.name
if (matched.isEmpty) {
- addError(s"Cannot find data for output column '${newColPath.quoted}'")
- None
+ throw
QueryCompilationErrors.incompatibleDataToTableCannotFindDataError(
Review Comment:
Do you still need the `addError` parameter?
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala:
##########
@@ -2076,12 +2076,114 @@ private[sql] object QueryCompilationErrors extends
QueryErrorsBase {
"dataColumns" -> query.output.map(c => s"'${c.name}'").mkString(", ")))
}
- def cannotWriteIncompatibleDataToTableError(tableName: String, errors:
Seq[String]): Throwable = {
+ def incompatibleDataToTableCannotFindDataError(
+ tableName: String, colPath: String): Throwable = {
new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1204",
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
messageParameters = Map(
- "tableName" -> tableName,
- "errors" -> errors.mkString("\n- ")))
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath)
+ )
+ )
+ }
+
+ def incompatibleDataToTableAmbiguousColumnNameError(
+ tableName: String, colPath: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.AMBIGUOUS_COLUMN_NAME",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath)
+ )
+ )
+ }
+
+ def incompatibleDataToTableExtraStructFieldsError(
+ tableName: String, colPath: String, extraCols: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath),
+ "extraCols" -> extraCols
+ )
+ )
+ }
+
+ def incompatibleDataToTableNullableColumnError(
+ tableName: String, colPath: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_COLUMN",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath)
+ )
+ )
+ }
+
+ def incompatibleDataToTableNullableArrayElementsError(
+ tableName: String, colPath: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_ARRAY_ELEMENTS",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath)
+ )
+ )
+ }
+
+ def incompatibleDataToTableNullableMapValuesError(
+ tableName: String, colPath: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_MAP_VALUES",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath)
+ )
+ )
+ }
+
+ def incompatibleDataToTableCannotSafelyCastError(
+ tableName: String, colPath: String, from: String, to: String): Throwable
= {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath),
+ "from" -> toSQLType(from),
+ "to" -> toSQLType(to)
+ )
+ )
+ }
+
+ def incompatibleDataToTableStructMissingFieldsError(
+ tableName: String, colPath: String, missingFields: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath),
+ "missingFields" -> missingFields
Review Comment:
Please, quote the field by `toSQLId()`.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala:
##########
@@ -2076,12 +2076,114 @@ private[sql] object QueryCompilationErrors extends
QueryErrorsBase {
"dataColumns" -> query.output.map(c => s"'${c.name}'").mkString(", ")))
}
- def cannotWriteIncompatibleDataToTableError(tableName: String, errors:
Seq[String]): Throwable = {
+ def incompatibleDataToTableCannotFindDataError(
+ tableName: String, colPath: String): Throwable = {
new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1204",
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
messageParameters = Map(
- "tableName" -> tableName,
- "errors" -> errors.mkString("\n- ")))
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath)
+ )
+ )
+ }
+
+ def incompatibleDataToTableAmbiguousColumnNameError(
+ tableName: String, colPath: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.AMBIGUOUS_COLUMN_NAME",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath)
+ )
+ )
+ }
+
+ def incompatibleDataToTableExtraStructFieldsError(
+ tableName: String, colPath: String, extraCols: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
+ messageParameters = Map(
+ "tableName" -> toSQLId(tableName),
+ "colPath" -> toSQLId(colPath),
+ "extraCols" -> extraCols
Review Comment:
This columns should be quoted via `toSQLId()` too.
##########
core/src/main/resources/error/error-classes.json:
##########
@@ -657,6 +657,59 @@
"Detected an incompatible DataSourceRegister. Please remove the
incompatible library from classpath or upgrade it. Error: <message>"
]
},
+ "INCOMPATIBLE_DATA_FOR_TABLE" : {
+ "message" : [
+ "Cannot write incompatible data for table <tableName>:"
+ ],
+ "subClass" : {
+ "AMBIGUOUS_COLUMN_NAME" : {
+ "message" : [
+ "Ambiguous column name in the input data: <colPath>."
+ ]
+ },
+ "CANNOT_FIND_DATA" : {
+ "message" : [
+ "Cannot find data for output column <colPath>."
+ ]
+ },
+ "CANNOT_SAFELY_CAST" : {
+ "message" : [
+ "Cannot safely cast <colPath>: <from> to <to>."
+ ]
+ },
+ "EXTRA_STRUCT_FIELDS" : {
+ "message" : [
+ "Cannot write extra fields to struct <colPath>: <extraCols>."
+ ]
+ },
+ "NULLABLE_ARRAY_ELEMENTS" : {
+ "message" : [
+ "Cannot write nullable elements to array of non-nulls: <colPath>."
+ ]
+ },
+ "NULLABLE_COLUMN" : {
+ "message" : [
+ "Cannot write nullable values to non-null column <colPath>."
+ ]
+ },
+ "NULLABLE_MAP_VALUES" : {
+ "message" : [
+ "Cannot write nullable elements to array of non-nulls: <colPath>."
Review Comment:
`to array`? Should map values, no?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
