This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a385e18432 [VL] Offload sequence with integer arguments to Velox
(#13503)
a385e18432 is described below
commit a385e18432c6a84c67b32ef4611df35541c37639
Author: Pedrum Jalali <[email protected]>
AuthorDate: Thu Nov 5 14:14:41 2026 -0800
[VL] Offload sequence with integer arguments to Velox (#13503)
Co-authored-by: Pedrum Jalali <[email protected]>
---
.../functions/ScalarFunctionsValidateSuite.scala | 65 ++++++++++++++++++++++
.../substrait/SubstraitToVeloxPlanValidator.cc | 52 ++++++++++++++++-
.../substrait/SubstraitToVeloxPlanValidator.h | 5 ++
.../velox/velox-backend-scalar-function-support.md | 56 +++++++++----------
4 files changed, 149 insertions(+), 29 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index e93b2c1316..774a80529a 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -559,6 +559,71 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ test("sequence with integer arguments") {
+ withTempPath {
+ path =>
+ Seq[(Integer, Integer)](
+ (1, 5),
+ (5, 1),
+ (3, 3),
+ (-2, 2),
+ (null, 4),
+ (2, null)
+ ).toDF("start", "stop")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare(
+ "SELECT sequence(start, stop), " +
+ "sequence(cast(start AS tinyint), cast(stop AS tinyint)), " +
+ "sequence(cast(start AS smallint), cast(stop AS smallint)), " +
+ "sequence(cast(start AS bigint), cast(stop AS bigint)) FROM tbl") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ runQueryAndCompare(
+ "SELECT sequence(start, stop, 2), " +
+ "sequence(cast(start AS tinyint), cast(stop AS tinyint), 2Y), " +
+ "sequence(cast(start AS bigint), cast(stop AS bigint), 3L) " +
+ "FROM tbl WHERE start <= stop") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ runQueryAndCompare(
+ "SELECT sequence(start, stop, -2), " +
+ "sequence(cast(start AS smallint), cast(stop AS smallint), -1S) " +
+ "FROM tbl WHERE start >= stop") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+ }
+
+ test("sequence with date arguments or a non-literal step falls back") {
+ withTempPath {
+ path =>
+ Seq[(Integer, Integer, Integer)](
+ (1, 5, 2),
+ (3, 3, 0)
+ ).toDF("start", "stop", "step")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare("SELECT sequence(start, stop, step) FROM tbl") {
+ checkSparkPlan[ProjectExec]
+ }
+ runQueryAndCompare("SELECT sequence(start, start, 0) FROM tbl") {
+ checkSparkPlan[ProjectExec]
+ }
+ runQueryAndCompare(
+ "SELECT sequence(date_add(DATE'2024-01-01', start), " +
+ "date_add(DATE'2024-01-01', stop)) FROM tbl") {
+ checkSparkPlan[ProjectExec]
+ }
+ }
+ }
+
test("map_from_arrays optimized by Spark constant folding") {
withSQLConf(("spark.sql.optimizer.excludedRules", "")) {
runQueryAndCompare("""SELECT map_from_arrays(sequence(1, 5),sequence(1,
5)), l_orderkey
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index 7ef0e807f1..c5e585be4d 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -56,7 +56,32 @@ const char* extractFileName(const char* file) {
const std::unordered_set<std::string> kRegexFunctions =
{"regexp_extract", "regexp_extract_all", "regexp_replace", "regexp_instr",
"rlike", "split"};
-const std::unordered_set<std::string> kBlackList = {"split_part", "sequence",
"approx_percentile"};
+const std::unordered_set<std::string> kBlackList = {"split_part",
"approx_percentile"};
+
+// DATE and the interval types share a TypeKind with the integer types, so
compare exact types.
+bool isIntegerType(const TypePtr& type) {
+ return TINYINT()->equivalent(*type) || SMALLINT()->equivalent(*type) ||
INTEGER()->equivalent(*type) ||
+ BIGINT()->equivalent(*type);
+}
+
+bool isNonZeroIntegerLiteral(const ::substrait::Expression& expression) {
+ if (!expression.has_literal()) {
+ return false;
+ }
+ const auto& literal = expression.literal();
+ switch (literal.literal_type_case()) {
+ case ::substrait::Expression_Literal::LiteralTypeCase::kI8:
+ return literal.i8() != 0;
+ case ::substrait::Expression_Literal::LiteralTypeCase::kI16:
+ return literal.i16() != 0;
+ case ::substrait::Expression_Literal::LiteralTypeCase::kI32:
+ return literal.i32() != 0;
+ case ::substrait::Expression_Literal::LiteralTypeCase::kI64:
+ return literal.i64() != 0;
+ default:
+ return false;
+ }
+}
} // namespace
bool SubstraitToVeloxPlanValidator::parseVeloxType(
@@ -195,6 +220,28 @@ bool SubstraitToVeloxPlanValidator::validateRegexExpr(
return true;
}
+bool SubstraitToVeloxPlanValidator::validateSequence(
+ const ::substrait::Expression::ScalarFunction& scalarFunction,
+ const std::vector<core::TypedExprPtr>& params) {
+ for (const auto& param : params) {
+ if (!isIntegerType(param->type())) {
+ LOG_VALIDATION_MSG(
+ "Function 'sequence' is not fully supported. Cause: sequence only
supports tinyint, smallint, integer and "
+ "bigint arguments; date and timestamp arguments are not supported in
Velox");
+ return false;
+ }
+ }
+ // Velox rejects a zero step even when start equals stop, where Spark
returns [start]. Only a literal step is
+ // known to be non-zero at planning time.
+ if (params.size() == 3 &&
!isNonZeroIntegerLiteral(scalarFunction.arguments()[2].value())) {
+ LOG_VALIDATION_MSG(
+ "Function 'sequence' is not fully supported. Cause: sequence only
supports an omitted step or a non-zero "
+ "literal step in Velox");
+ return false;
+ }
+ return true;
+}
+
bool SubstraitToVeloxPlanValidator::validateScalarFunction(
const ::substrait::Expression::ScalarFunction& scalarFunction,
const RowTypePtr& inputType) {
@@ -218,6 +265,9 @@ bool SubstraitToVeloxPlanValidator::validateScalarFunction(
if (name == "extract") {
return validateExtractExpr(params);
}
+ if (name == "sequence") {
+ return validateSequence(scalarFunction, params);
+ }
// Validate regex functions.
if (kRegexFunctions.find(name) != kRegexFunctions.end()) {
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h
index cb8b322301..2cb7518e84 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h
@@ -160,6 +160,11 @@ class SubstraitToVeloxPlanValidator {
/// Check if the pattern can pass with RE2 compilation.
bool validateRegexExpr(const std::string& name, const
::substrait::Expression::ScalarFunction& scalarFunction);
+ /// Validate sequence function. Only integer arguments, with no step or a
non-zero literal step, are supported.
+ bool validateSequence(
+ const ::substrait::Expression::ScalarFunction& scalarFunction,
+ const std::vector<core::TypedExprPtr>& params);
+
/// Validate Substrait scarlar function.
bool validateScalarFunction(
const ::substrait::Expression::ScalarFunction& scalarFunction,
diff --git a/docs/backends/velox/velox-backend-scalar-function-support.md
b/docs/backends/velox/velox-backend-scalar-function-support.md
index 549c7f1b45..404b63442c 100644
--- a/docs/backends/velox/velox-backend-scalar-function-support.md
+++ b/docs/backends/velox/velox-backend-scalar-function-support.md
@@ -1,39 +1,39 @@
# Scalar Functions Support Status
-**Out of 433 scalar functions in Spark 4.1, Gluten currently fully supports
261 functions and partially supports 31 functions.**
+**Out of 433 scalar functions in Spark 4.1, Gluten currently fully supports
261 functions and partially supports 32 functions.**
The status applies to `spark.sql.ansi.enabled=false`. When ANSI mode is
enabled, Gluten falls back to vanilla Spark
(see `spark.gluten.sql.ansiFallback.enabled`).
## Array Functions
-| Spark Functions | Spark Expressions | Status | Restrictions |
-|-------------------|---------------------|----------|----------------|
-| array | CreateArray | S | |
-| array_append | ArrayAppend | S | |
-| array_compact | ArrayCompact | S | |
-| array_contains | ArrayContains | S | |
-| array_distinct | ArrayDistinct | S | |
-| array_except | ArrayExcept | S | |
-| array_insert | ArrayInsert | S | |
-| array_intersect | ArrayIntersect | S | |
-| array_join | ArrayJoin | S | |
-| array_max | ArrayMax | S | |
-| array_min | ArrayMin | S | |
-| array_position | ArrayPosition | S | |
-| array_prepend | ArrayPrepend | S | |
-| array_remove | ArrayRemove | S | |
-| array_repeat | ArrayRepeat | S | |
-| array_size | ArraySize | S | |
-| array_union | ArrayUnion | S | |
-| arrays_overlap | ArraysOverlap | S | |
-| arrays_zip | ArraysZip | S | |
-| flatten | Flatten | S | |
-| get | Get | S | |
-| sequence | Sequence | | |
-| shuffle | Shuffle | S | |
-| slice | Slice | S | |
-| sort_array | SortArray | S | |
+| Spark Functions | Spark Expressions | Status | Restrictions
|
+|-------------------|---------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| array | CreateArray | S |
|
+| array_append | ArrayAppend | S |
|
+| array_compact | ArrayCompact | S |
|
+| array_contains | ArrayContains | S |
|
+| array_distinct | ArrayDistinct | S |
|
+| array_except | ArrayExcept | S |
|
+| array_insert | ArrayInsert | S |
|
+| array_intersect | ArrayIntersect | S |
|
+| array_join | ArrayJoin | S |
|
+| array_max | ArrayMax | S |
|
+| array_min | ArrayMin | S |
|
+| array_position | ArrayPosition | S |
|
+| array_prepend | ArrayPrepend | S |
|
+| array_remove | ArrayRemove | S |
|
+| array_repeat | ArrayRepeat | S |
|
+| array_size | ArraySize | S |
|
+| array_union | ArrayUnion | S |
|
+| arrays_overlap | ArraysOverlap | S |
|
+| arrays_zip | ArraysZip | S |
|
+| flatten | Flatten | S |
|
+| get | Get | S |
|
+| sequence | Sequence | PS | sequence only supports
an omitted step or a non-zero literal step in Velox<br>sequence only supports
tinyint, smallint, integer and bigint arguments; date and timestamp arguments
are not supported in Velox |
+| shuffle | Shuffle | S |
|
+| slice | Slice | S |
|
+| sort_array | SortArray | S |
|
## Bitwise Functions
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
