This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new ffa71322830a [SPARK-57576][SQL] Fix `SET_OPERATION_ON_MAP_TYPE` and
`SET_OPERATION_ON_VARIANT_TYPE` exception messages
ffa71322830a is described below
commit ffa71322830ac95e26841747c3589f030000cfec
Author: Mihailo Aleksic <[email protected]>
AuthorDate: Mon Jun 22 14:32:38 2026 +0200
[SPARK-57576][SQL] Fix `SET_OPERATION_ON_MAP_TYPE` and
`SET_OPERATION_ON_VARIANT_TYPE` exception messages
### What changes were proposed in this pull request?
`SET_OPERATION_ON_MAP_TYPE` exception message references Dataframe programs
only although it's thrown for SQL queries as well. Same stands for
`SET_OPERATION_ON_VARIANT_TYPE`. In this PR I propose to fix it.
### Why are the changes needed?
To improve error message.
### Does this PR introduce _any_ user-facing change?
Yes, improved error message.
### How was this patch tested?
Existing + added tests.
### Was this patch authored or co-authored using generative AI tooling?
Yes.
Closes #56611 from mihailoale-db/fixerromsg2.
Authored-by: Mihailo Aleksic <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
(cherry picked from commit 5d2a9b12a3abe502b543b4b7152bcedbf8682791)
Signed-off-by: Max Gekk <[email protected]>
---
.../src/main/resources/error/error-conditions.json | 4 +-
.../set-operations-unsupported-types.sql.out | 246 +++++++++++++++++++
.../inputs/set-operations-unsupported-types.sql | 36 +++
.../set-operations-unsupported-types.sql.out | 268 +++++++++++++++++++++
4 files changed, 552 insertions(+), 2 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json
b/common/utils/src/main/resources/error/error-conditions.json
index 94634a77befc..3ccaa3bd94bd 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -8364,12 +8364,12 @@
},
"SET_OPERATION_ON_MAP_TYPE" : {
"message" : [
- "Cannot have MAP type columns in DataFrame which calls set
operations (INTERSECT, EXCEPT, etc.), but the type of column <colName> is
<dataType>."
+ "Cannot have MAP type columns in a set operation (INTERSECT, EXCEPT,
etc.), but the type of column <colName> is <dataType>."
]
},
"SET_OPERATION_ON_VARIANT_TYPE" : {
"message" : [
- "Cannot have VARIANT type columns in DataFrame which calls set
operations (INTERSECT, EXCEPT, etc.), but the type of column <colName> is
<dataType>."
+ "Cannot have VARIANT type columns in a set operation (INTERSECT,
EXCEPT, etc.), but the type of column <colName> is <dataType>."
]
},
"SET_PATH_WHEN_DISABLED" : {
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/set-operations-unsupported-types.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/set-operations-unsupported-types.sql.out
new file mode 100644
index 000000000000..d785d33e2df3
--- /dev/null
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/set-operations-unsupported-types.sql.out
@@ -0,0 +1,246 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+create temporary view map_view as select map(1, 2) as m, id from range(2)
+-- !query analysis
+CreateViewCommand `map_view`, select map(1, 2) as m, id from range(2), false,
false, LocalTempView, UNSUPPORTED, true
+ +- Project [map(1, 2) AS m#x, id#xL]
+ +- Range (0, 2, step=1)
+
+
+-- !query
+create temporary view variant_view as select parse_json('1') as v, id from
range(2)
+-- !query analysis
+CreateViewCommand `variant_view`, select parse_json('1') as v, id from
range(2), false, false, LocalTempView, UNSUPPORTED, true
+ +- Project [parse_json(1, true) AS v#x, id#xL]
+ +- Range (0, 2, step=1)
+
+
+-- !query
+SELECT * FROM map_view INTERSECT SELECT * FROM map_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`m`",
+ "dataType" : "\"MAP<INT, INT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 55,
+ "fragment" : "SELECT * FROM map_view INTERSECT SELECT * FROM map_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM map_view EXCEPT SELECT * FROM map_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`m`",
+ "dataType" : "\"MAP<INT, INT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 52,
+ "fragment" : "SELECT * FROM map_view EXCEPT SELECT * FROM map_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM map_view UNION SELECT * FROM map_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`m`",
+ "dataType" : "\"MAP<INT, INT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 51,
+ "fragment" : "SELECT * FROM map_view UNION SELECT * FROM map_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT m FROM map_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`m`",
+ "dataType" : "\"MAP<INT, INT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 31,
+ "fragment" : "SELECT DISTINCT m FROM map_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`v`",
+ "dataType" : "\"VARIANT\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 63,
+ "fragment" : "SELECT * FROM variant_view INTERSECT SELECT * FROM
variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`v`",
+ "dataType" : "\"VARIANT\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 60,
+ "fragment" : "SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view UNION SELECT * FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`v`",
+ "dataType" : "\"VARIANT\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 59,
+ "fragment" : "SELECT * FROM variant_view UNION SELECT * FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT v FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`v`",
+ "dataType" : "\"VARIANT\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 35,
+ "fragment" : "SELECT DISTINCT v FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT struct(v) FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`struct(v)`",
+ "dataType" : "\"STRUCT<v: VARIANT NOT NULL>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 43,
+ "fragment" : "SELECT DISTINCT struct(v) FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT array(v) FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`array(v)`",
+ "dataType" : "\"ARRAY<VARIANT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 42,
+ "fragment" : "SELECT DISTINCT array(v) FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT map('m', v) FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`map(m, v)`",
+ "dataType" : "\"MAP<STRING, VARIANT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 45,
+ "fragment" : "SELECT DISTINCT map('m', v) FROM variant_view"
+ } ]
+}
diff --git
a/sql/core/src/test/resources/sql-tests/inputs/set-operations-unsupported-types.sql
b/sql/core/src/test/resources/sql-tests/inputs/set-operations-unsupported-types.sql
new file mode 100644
index 000000000000..8e9ef6af63e6
--- /dev/null
+++
b/sql/core/src/test/resources/sql-tests/inputs/set-operations-unsupported-types.sql
@@ -0,0 +1,36 @@
+-- Tests that set operations (and the DISTINCT/de-duplication they rely on) are
+-- rejected when a column has MAP or VARIANT type, since those types are not
+-- orderable. See UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE and
+-- UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE.
+
+create temporary view map_view as select map(1, 2) as m, id from range(2);
+
+create temporary view variant_view as select parse_json('1') as v, id from
range(2);
+
+
+-- MAP type columns in set operations
+SELECT * FROM map_view INTERSECT SELECT * FROM map_view;
+
+SELECT * FROM map_view EXCEPT SELECT * FROM map_view;
+
+SELECT * FROM map_view UNION SELECT * FROM map_view;
+
+SELECT DISTINCT m FROM map_view;
+
+
+-- VARIANT type columns in set operations
+SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view;
+
+SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view;
+
+SELECT * FROM variant_view UNION SELECT * FROM variant_view;
+
+SELECT DISTINCT v FROM variant_view;
+
+
+-- VARIANT nested inside complex types is also rejected
+SELECT DISTINCT struct(v) FROM variant_view;
+
+SELECT DISTINCT array(v) FROM variant_view;
+
+SELECT DISTINCT map('m', v) FROM variant_view;
diff --git
a/sql/core/src/test/resources/sql-tests/results/set-operations-unsupported-types.sql.out
b/sql/core/src/test/resources/sql-tests/results/set-operations-unsupported-types.sql.out
new file mode 100644
index 000000000000..80fde871121f
--- /dev/null
+++
b/sql/core/src/test/resources/sql-tests/results/set-operations-unsupported-types.sql.out
@@ -0,0 +1,268 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+create temporary view map_view as select map(1, 2) as m, id from range(2)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+create temporary view variant_view as select parse_json('1') as v, id from
range(2)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT * FROM map_view INTERSECT SELECT * FROM map_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`m`",
+ "dataType" : "\"MAP<INT, INT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 55,
+ "fragment" : "SELECT * FROM map_view INTERSECT SELECT * FROM map_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM map_view EXCEPT SELECT * FROM map_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`m`",
+ "dataType" : "\"MAP<INT, INT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 52,
+ "fragment" : "SELECT * FROM map_view EXCEPT SELECT * FROM map_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM map_view UNION SELECT * FROM map_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`m`",
+ "dataType" : "\"MAP<INT, INT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 51,
+ "fragment" : "SELECT * FROM map_view UNION SELECT * FROM map_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT m FROM map_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`m`",
+ "dataType" : "\"MAP<INT, INT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 31,
+ "fragment" : "SELECT DISTINCT m FROM map_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`v`",
+ "dataType" : "\"VARIANT\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 63,
+ "fragment" : "SELECT * FROM variant_view INTERSECT SELECT * FROM
variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`v`",
+ "dataType" : "\"VARIANT\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 60,
+ "fragment" : "SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view UNION SELECT * FROM variant_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`v`",
+ "dataType" : "\"VARIANT\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 59,
+ "fragment" : "SELECT * FROM variant_view UNION SELECT * FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT v FROM variant_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`v`",
+ "dataType" : "\"VARIANT\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 35,
+ "fragment" : "SELECT DISTINCT v FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT struct(v) FROM variant_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`struct(v)`",
+ "dataType" : "\"STRUCT<v: VARIANT NOT NULL>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 43,
+ "fragment" : "SELECT DISTINCT struct(v) FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT array(v) FROM variant_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`array(v)`",
+ "dataType" : "\"ARRAY<VARIANT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 42,
+ "fragment" : "SELECT DISTINCT array(v) FROM variant_view"
+ } ]
+}
+
+
+-- !query
+SELECT DISTINCT map('m', v) FROM variant_view
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+ "sqlState" : "0A000",
+ "messageParameters" : {
+ "colName" : "`map(m, v)`",
+ "dataType" : "\"MAP<STRING, VARIANT>\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 1,
+ "stopIndex" : 45,
+ "fragment" : "SELECT DISTINCT map('m', v) FROM variant_view"
+ } ]
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
