This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new fdb130f690 [GLUTEN-11425][CORE] Upgrade Spark 4.1.0 to 4.1.1 (#11426)
fdb130f690 is described below
commit fdb130f690df44475f5a1e15e2bb8edbee40b427
Author: Chang Chen <[email protected]>
AuthorDate: Tue Jan 20 09:06:12 2026 +0800
[GLUTEN-11425][CORE] Upgrade Spark 4.1.0 to 4.1.1 (#11426)
* [CORE] Upgrade Spark 4.1.0 to 4.1.1
* update spark 411 test
* Remove exclusion for SPARK-53413 in VeloxTestSettings
---
.github/workflows/util/install-spark-resources.sh | 2 +-
.../sql-tests/inputs/identifier-clause.sql | 6 +
.../sql-tests/inputs/kllquantiles.sql | 57 ++++++--
.../results/identifier-clause-legacy.sql.out | 48 +++++++
.../sql-tests/results/identifier-clause.sql.out | 57 +++++---
.../sql-tests/results/kllquantiles.sql.out | 154 ++++++++++++++++++---
.../gluten/utils/velox/VeloxTestSettings.scala | 2 -
pom.xml | 2 +-
tools/gluten-it/pom.xml | 2 +-
9 files changed, 280 insertions(+), 50 deletions(-)
diff --git a/.github/workflows/util/install-spark-resources.sh
b/.github/workflows/util/install-spark-resources.sh
index d245dbb4ac..c2cd268f72 100755
--- a/.github/workflows/util/install-spark-resources.sh
+++ b/.github/workflows/util/install-spark-resources.sh
@@ -122,7 +122,7 @@ case "$1" in
4.1)
# Spark-4.x, scala 2.12 // using 2.12 as a hack as 4.0 does not have 2.13
suffix
cd ${INSTALL_DIR} && \
- install_spark "4.1.0" "3" "2.12"
+ install_spark "4.1.1" "3" "2.12"
;;
*)
echo "Spark version is expected to be specified."
diff --git
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/inputs/identifier-clause.sql
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/inputs/identifier-clause.sql
index bb6c710706..d9bafe7cc6 100644
---
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/inputs/identifier-clause.sql
+++
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/inputs/identifier-clause.sql
@@ -367,6 +367,12 @@ SELECT * FROM unpivot_test UNPIVOT (val FOR col IN (a AS
IDENTIFIER('col_a'), b
SELECT * FROM unpivot_test UNPIVOT ((v1, v2) FOR col IN ((a, b) AS
IDENTIFIER('cols_ab'), (b, c) AS IDENTIFIER('cols_bc'))) ORDER BY ALL;
DROP TABLE unpivot_test;
+-- DESCRIBE column with IDENTIFIER()
+CREATE TABLE describe_col_test(c1 INT, c2 STRING, c3 DOUBLE) USING CSV;
+DESCRIBE describe_col_test IDENTIFIER('c1');
+DESCRIBE describe_col_test IDENTIFIER('c2');
+DROP TABLE describe_col_test;
+
-- All the following tests fail because they are not about "true" identifiers
-- This should fail - named parameters don't support IDENTIFIER()
diff --git
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/inputs/kllquantiles.sql
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/inputs/kllquantiles.sql
index d0d7fb1f9c..fe1b61de03 100644
---
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/inputs/kllquantiles.sql
+++
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/inputs/kllquantiles.sql
@@ -64,7 +64,7 @@ VALUES
(CAST(7 AS DOUBLE), CAST(11 AS DOUBLE)) AS tab(col1, col2);
-- BIGINT sketches
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -72,7 +72,7 @@ FROM (
FROM t_byte_1_5_through_7_11
);
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -80,7 +80,7 @@ FROM (
FROM t_int_1_5_through_7_11
);
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -88,7 +88,7 @@ FROM (
FROM t_long_1_5_through_7_11
);
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -97,7 +97,7 @@ FROM (
);
-- FLOAT sketches (only accepts float types to avoid precision loss)
-SELECT split(kll_sketch_to_string_float(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_float(agg)) LIKE '%kll%' AS str_contains_kll,
abs(kll_sketch_get_quantile_float(agg, 0.5) - 4.0) < 0.5 AS
median_close_to_4,
abs(kll_sketch_get_rank_float(agg, 3) - 0.4) < 0.1 AS rank3_close_to_0_4
FROM (
@@ -106,7 +106,7 @@ FROM (
);
-- DOUBLE sketches (accepts float and double types to avoid precision loss
from integer conversion)
-SELECT split(kll_sketch_to_string_double(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_double(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_double(agg, 0.5) - 4.0) < 0.5 AS
median_close_to_4,
abs(kll_sketch_get_rank_double(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -115,7 +115,7 @@ FROM (
);
-- Test float column with double sketch (valid type promotion)
-SELECT split(kll_sketch_to_string_double(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_double(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_double(agg, 0.5) - 4.0) < 0.5 AS
median_close_to_4,
abs(kll_sketch_get_rank_double(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -268,7 +268,7 @@ FROM t_long_1_5_through_7_11;
-- These queries should fail with type mismatch or validation errors
-- Type mismatch: BIGINT sketch does not accept DOUBLE columns
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -277,7 +277,7 @@ FROM (
);
-- Type mismatch: BIGINT sketch does not accept FLOAT columns
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -286,7 +286,7 @@ FROM (
);
-- Type mismatch: FLOAT sketch does not accept DOUBLE columns
-SELECT split(kll_sketch_to_string_float(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_float(agg)) LIKE '%kll%' AS str_contains_kll,
abs(kll_sketch_get_quantile_float(agg, 0.5) - 4.0) < 0.5 AS
median_close_to_4,
abs(kll_sketch_get_rank_float(agg, 3) - 0.4) < 0.1 AS rank3_close_to_0_4
FROM (
@@ -378,7 +378,7 @@ FROM (
-- interpret the binary data. This query succeeds even though we're using a
DOUBLE
-- to_string function on a BIGINT sketch. The function reads the binary
representation
-- and produces output, but the numeric values will be incorrectly interpreted.
-SELECT kll_sketch_to_string_double(agg) LIKE '%Kll%' AS contains_kll_header
+SELECT lower(kll_sketch_to_string_double(agg)) LIKE '%kll%' AS
contains_kll_header
FROM (
SELECT kll_sketch_agg_bigint(col1) AS agg
FROM t_long_1_5_through_7_11
@@ -464,6 +464,41 @@ FROM (
FROM t_double_1_5_through_7_11
);
+-- Negative tests for non-foldable (non-constant) rank/quantile arguments
+-- These tests verify that get_quantile and get_rank functions require
compile-time constant arguments
+
+-- Non-foldable scalar rank argument to get_quantile (column reference)
+SELECT kll_sketch_get_quantile_bigint(agg, CAST(col1 AS DOUBLE) / 10.0) AS
non_foldable_scalar_rank
+FROM (
+ SELECT kll_sketch_agg_bigint(col1) AS agg, col1
+ FROM t_long_1_5_through_7_11
+ GROUP BY col1
+);
+
+-- Non-foldable array rank argument to get_quantile (array containing column
reference)
+SELECT kll_sketch_get_quantile_bigint(agg, array(0.25, CAST(col1 AS DOUBLE) /
10.0, 0.75)) AS non_foldable_array_rank
+FROM (
+ SELECT kll_sketch_agg_bigint(col1) AS agg, col1
+ FROM t_long_1_5_through_7_11
+ GROUP BY col1
+);
+
+-- Non-foldable scalar quantile argument to get_rank (column reference)
+SELECT kll_sketch_get_rank_bigint(agg, col1) AS non_foldable_scalar_quantile
+FROM (
+ SELECT kll_sketch_agg_bigint(col1) AS agg, col1
+ FROM t_long_1_5_through_7_11
+ GROUP BY col1
+);
+
+-- Non-foldable array quantile argument to get_rank (array containing column
reference)
+SELECT kll_sketch_get_rank_bigint(agg, array(1L, col1, 5L)) AS
non_foldable_array_quantile
+FROM (
+ SELECT kll_sketch_agg_bigint(col1) AS agg, col1
+ FROM t_long_1_5_through_7_11
+ GROUP BY col1
+);
+
-- Clean up
DROP TABLE IF EXISTS t_int_1_5_through_7_11;
DROP TABLE IF EXISTS t_long_1_5_through_7_11;
diff --git
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/identifier-clause-legacy.sql.out
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/identifier-clause-legacy.sql.out
index 272d9bce81..6a99be0570 100644
---
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/identifier-clause-legacy.sql.out
+++
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/identifier-clause-legacy.sql.out
@@ -2740,6 +2740,54 @@ struct<>
+-- !query
+CREATE TABLE describe_col_test(c1 INT, c2 STRING, c3 DOUBLE) USING CSV
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DESCRIBE describe_col_test IDENTIFIER('c1')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+{
+ "errorClass" : "PARSE_SYNTAX_ERROR",
+ "sqlState" : "42601",
+ "messageParameters" : {
+ "error" : "'('",
+ "hint" : ""
+ }
+}
+
+
+-- !query
+DESCRIBE describe_col_test IDENTIFIER('c2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+{
+ "errorClass" : "PARSE_SYNTAX_ERROR",
+ "sqlState" : "42601",
+ "messageParameters" : {
+ "error" : "'('",
+ "hint" : ""
+ }
+}
+
+
+-- !query
+DROP TABLE describe_col_test
+-- !query schema
+struct<>
+-- !query output
+
+
+
-- !query
SELECT :IDENTIFIER('param1') FROM VALUES(1) AS T(c1)
-- !query schema
diff --git
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/identifier-clause.sql.out
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/identifier-clause.sql.out
index b398c07b14..0c04737912 100644
---
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/identifier-clause.sql.out
+++
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/identifier-clause.sql.out
@@ -2227,24 +2227,11 @@ struct<result:int>
EXECUTE IMMEDIATE 'SELECT IDENTIFIER(:alias ''.c1'') FROM integration_test AS
IDENTIFIER(:alias) ORDER BY ALL'
USING 't' AS alias
-- !query schema
-struct<>
+struct<c1:int>
-- !query output
-org.apache.spark.sql.catalyst.ExtendedAnalysisException
-{
- "errorClass" : "UNRESOLVED_COLUMN.WITH_SUGGESTION",
- "sqlState" : "42703",
- "messageParameters" : {
- "objectName" : "`t`.`c1`",
- "proposal" : "`IDENTIFIER('t')`.`c1`, `IDENTIFIER('t')`.`c2`,
`IDENTIFIER('t')`.`c4`"
- },
- "queryContext" : [ {
- "objectType" : "EXECUTE IMMEDIATE",
- "objectName" : "",
- "startIndex" : 8,
- "stopIndex" : 31,
- "fragment" : "IDENTIFIER(:alias '.c1')"
- } ]
-}
+1
+2
+3
-- !query
@@ -2396,6 +2383,42 @@ struct<>
+-- !query
+CREATE TABLE describe_col_test(c1 INT, c2 STRING, c3 DOUBLE) USING CSV
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DESCRIBE describe_col_test IDENTIFIER('c1')
+-- !query schema
+struct<info_name:string,info_value:string>
+-- !query output
+col_name c1
+data_type int
+comment NULL
+
+
+-- !query
+DESCRIBE describe_col_test IDENTIFIER('c2')
+-- !query schema
+struct<info_name:string,info_value:string>
+-- !query output
+col_name c2
+data_type string
+comment NULL
+
+
+-- !query
+DROP TABLE describe_col_test
+-- !query schema
+struct<>
+-- !query output
+
+
+
-- !query
SELECT :IDENTIFIER('param1') FROM VALUES(1) AS T(c1)
-- !query schema
diff --git
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/kllquantiles.sql.out
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/kllquantiles.sql.out
index 3618c85193..863bded159 100644
---
a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/kllquantiles.sql.out
+++
b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/kllquantiles.sql.out
@@ -134,7 +134,7 @@ struct<>
-- !query
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -148,7 +148,7 @@ true true true
-- !query
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -162,7 +162,7 @@ true true true
-- !query
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -176,7 +176,7 @@ true true true
-- !query
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -190,7 +190,7 @@ true true true
-- !query
-SELECT split(kll_sketch_to_string_float(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_float(agg)) LIKE '%kll%' AS str_contains_kll,
abs(kll_sketch_get_quantile_float(agg, 0.5) - 4.0) < 0.5 AS
median_close_to_4,
abs(kll_sketch_get_rank_float(agg, 3) - 0.4) < 0.1 AS rank3_close_to_0_4
FROM (
@@ -204,7 +204,7 @@ true true true
-- !query
-SELECT split(kll_sketch_to_string_double(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_double(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_double(agg, 0.5) - 4.0) < 0.5 AS
median_close_to_4,
abs(kll_sketch_get_rank_double(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -218,7 +218,7 @@ true true true
-- !query
-SELECT split(kll_sketch_to_string_double(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_double(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_double(agg, 0.5) - 4.0) < 0.5 AS
median_close_to_4,
abs(kll_sketch_get_rank_double(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -512,7 +512,7 @@ struct<n_k_100:bigint>
-- !query
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -536,15 +536,15 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "startIndex" : 273,
- "stopIndex" : 299,
+ "startIndex" : 264,
+ "stopIndex" : 290,
"fragment" : "kll_sketch_agg_bigint(col1)"
} ]
}
-- !query
-SELECT split(kll_sketch_to_string_bigint(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_bigint(agg)) LIKE '%kll%' AS
str_contains_kll,
abs(kll_sketch_get_quantile_bigint(agg, 0.5) - 4) < 1 AS
median_close_to_4,
abs(kll_sketch_get_rank_bigint(agg, 3) - 0.4) < 0.1 AS
rank3_close_to_0_4
FROM (
@@ -568,15 +568,15 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "startIndex" : 273,
- "stopIndex" : 299,
+ "startIndex" : 264,
+ "stopIndex" : 290,
"fragment" : "kll_sketch_agg_bigint(col1)"
} ]
}
-- !query
-SELECT split(kll_sketch_to_string_float(agg), '\n')[1] LIKE '%Kll%' AS
str_contains_kll,
+SELECT lower(kll_sketch_to_string_float(agg)) LIKE '%kll%' AS str_contains_kll,
abs(kll_sketch_get_quantile_float(agg, 0.5) - 4.0) < 0.5 AS
median_close_to_4,
abs(kll_sketch_get_rank_float(agg, 3) - 0.4) < 0.1 AS rank3_close_to_0_4
FROM (
@@ -600,8 +600,8 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "startIndex" : 274,
- "stopIndex" : 299,
+ "startIndex" : 265,
+ "stopIndex" : 290,
"fragment" : "kll_sketch_agg_float(col1)"
} ]
}
@@ -953,7 +953,7 @@ true
-- !query
-SELECT kll_sketch_to_string_double(agg) LIKE '%Kll%' AS contains_kll_header
+SELECT lower(kll_sketch_to_string_double(agg)) LIKE '%kll%' AS
contains_kll_header
FROM (
SELECT kll_sketch_agg_bigint(col1) AS agg
FROM t_long_1_5_through_7_11
@@ -1395,6 +1395,126 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+SELECT kll_sketch_get_quantile_bigint(agg, CAST(col1 AS DOUBLE) / 10.0) AS
non_foldable_scalar_rank
+FROM (
+ SELECT kll_sketch_agg_bigint(col1) AS agg, col1
+ FROM t_long_1_5_through_7_11
+ GROUP BY col1
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "inputExpr" : "\"(CAST(col1 AS DOUBLE) / 10.0)\"",
+ "inputName" : "`rank`",
+ "inputType" : "\"DOUBLE\"",
+ "sqlExpr" : "\"kll_sketch_get_quantile_bigint(agg, (CAST(col1 AS DOUBLE) /
10.0))\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 71,
+ "fragment" : "kll_sketch_get_quantile_bigint(agg, CAST(col1 AS DOUBLE) /
10.0)"
+ } ]
+}
+
+
+-- !query
+SELECT kll_sketch_get_quantile_bigint(agg, array(0.25, CAST(col1 AS DOUBLE) /
10.0, 0.75)) AS non_foldable_array_rank
+FROM (
+ SELECT kll_sketch_agg_bigint(col1) AS agg, col1
+ FROM t_long_1_5_through_7_11
+ GROUP BY col1
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "inputExpr" : "\"array(0.25, (CAST(col1 AS DOUBLE) / 10.0), 0.75)\"",
+ "inputName" : "`rank`",
+ "inputType" : "\"ARRAY<DOUBLE>\"",
+ "sqlExpr" : "\"kll_sketch_get_quantile_bigint(agg, array(0.25, (CAST(col1
AS DOUBLE) / 10.0), 0.75))\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 90,
+ "fragment" : "kll_sketch_get_quantile_bigint(agg, array(0.25, CAST(col1 AS
DOUBLE) / 10.0, 0.75))"
+ } ]
+}
+
+
+-- !query
+SELECT kll_sketch_get_rank_bigint(agg, col1) AS non_foldable_scalar_quantile
+FROM (
+ SELECT kll_sketch_agg_bigint(col1) AS agg, col1
+ FROM t_long_1_5_through_7_11
+ GROUP BY col1
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "inputExpr" : "\"col1\"",
+ "inputName" : "`quantile`",
+ "inputType" : "\"BIGINT\"",
+ "sqlExpr" : "\"kll_sketch_get_rank_bigint(agg, col1)\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 44,
+ "fragment" : "kll_sketch_get_rank_bigint(agg, col1)"
+ } ]
+}
+
+
+-- !query
+SELECT kll_sketch_get_rank_bigint(agg, array(1L, col1, 5L)) AS
non_foldable_array_quantile
+FROM (
+ SELECT kll_sketch_agg_bigint(col1) AS agg, col1
+ FROM t_long_1_5_through_7_11
+ GROUP BY col1
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "inputExpr" : "\"array(1, col1, 5)\"",
+ "inputName" : "`quantile`",
+ "inputType" : "\"ARRAY<BIGINT>\"",
+ "sqlExpr" : "\"kll_sketch_get_rank_bigint(agg, array(1, col1, 5))\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 59,
+ "fragment" : "kll_sketch_get_rank_bigint(agg, array(1L, col1, 5L))"
+ } ]
+}
+
+
-- !query
DROP TABLE IF EXISTS t_int_1_5_through_7_11
-- !query schema
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 5898a7728f..29cd082b45 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1016,8 +1016,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("dumping query execution info to a file - explainMode=formatted")
// TODO: fix in Spark-4.0
.exclude("SPARK-47289: extended explain info")
- // TODO: fix on Spark-4.1 introduced by
https://github.com/apache/spark/pull/52157
- .exclude("SPARK-53413: Cleanup shuffle dependencies for commands")
override def getSQLQueryTestSettings: SQLQueryTestSettings =
VeloxSQLQueryTestSettings
}
// scalastyle:on line.size.limit
diff --git a/pom.xml b/pom.xml
index 5bd520a462..cb7a036f6d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1278,7 +1278,7 @@
<properties>
<sparkbundle.version>4.1</sparkbundle.version>
<sparkshim.artifactId>spark-sql-columnar-shims-spark41</sparkshim.artifactId>
- <spark.version>4.1.0</spark.version>
+ <spark.version>4.1.1</spark.version>
<iceberg.version>1.10.0</iceberg.version>
<delta.package.name>delta-spark</delta.package.name>
<delta.version>4.0.0</delta.version>
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index 77e4eab9f6..0020ca9f99 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -330,7 +330,7 @@
<profile>
<id>spark-4.1</id>
<properties>
- <spark.version>4.1.0</spark.version>
+ <spark.version>4.1.1</spark.version>
<scala.version>2.13.17</scala.version>
<scala.binary.version>2.13</scala.binary.version>
<delta.package.name>delta-spark</delta.package.name>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
