This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a5ceb3793b [GLUTEN-7628][VL] Support monotonically_increasing_id
function (#13268)
a5ceb3793b is described below
commit a5ceb3793b7d22be5ec777ae322db3838d9fee67
Author: rdtr <[email protected]>
AuthorDate: Fri Oct 16 20:51:01 2026 -0700
[GLUTEN-7628][VL] Support monotonically_increasing_id function (#13268)
---
.../apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala | 1 +
.../apache/gluten/functions/ScalarFunctionsValidateSuite.scala | 8 ++++++--
cpp/velox/compute/WholeStageResultIterator.cc | 4 ++++
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index 640e42690b..eae0ff97f6 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -1404,6 +1404,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with
Logging {
Sig[VeloxBloomFilterAggregate](ExpressionNames.BLOOM_FILTER_AGG),
Sig[MapFilter](ExpressionNames.MAP_FILTER),
Sig[AssertNotNull](ExpressionNames.ASSERT_NOT_NULL),
+
Sig[MonotonicallyIncreasingID](ExpressionNames.MONOTONICALLY_INCREASING_ID),
// For test purpose.
Sig[VeloxDummyExpression](VeloxDummyExpression.VELOX_DUMMY_EXPRESSION)
)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 458ca01b8c..c454db85a2 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -538,12 +538,16 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- // FIXME: Ignored: https://github.com/apache/gluten/issues/7600.
- ignore("monotonically_increasintestg_id") {
+ test("monotonically_increasing_id") {
runQueryAndCompare("""SELECT monotonically_increasing_id(), l_orderkey
| from lineitem limit 100""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
+ // Each call keeps its own state, so both columns have the same values
(#7628).
+ runQueryAndCompare("""SELECT monotonically_increasing_id(),
monotonically_increasing_id()
+ | from lineitem limit 100""".stripMargin) {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
}
test("sequence function optimized by Spark constant folding") {
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc
b/cpp/velox/compute/WholeStageResultIterator.cc
index fab69bab1a..5fd4f8b878 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -650,6 +650,10 @@ std::unordered_map<std::string, std::string>
WholeStageResultIterator::getQueryC
configs[SparkQueryConfig::qualify(SparkQueryConfig::kPartitionId)] =
std::to_string(taskInfo_.partitionId);
+ // Spark keeps each call of a non-deterministic function separate, with
its own state. Without
+ // this, Velox merges identical calls such as two
monotonically_increasing_id() into one.
+ configs[velox::core::QueryConfig::kExprDedupNonDeterministic] = "false";
+
// Enable Spark legacy date formatter if spark.sql.legacy.timeParserPolicy
is set to 'LEGACY'
// or 'legacy'
if (veloxCfg_->get<std::string>(kSparkLegacyTimeParserPolicy, "") ==
"LEGACY") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
