This is an automated email from the ASF dual-hosted git repository.
rui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f37b0dd224 [GLUTEN-6887][VL] Daily Update Velox Version (2025_09_11)
(#10685)
f37b0dd224 is described below
commit f37b0dd2240f6182f501d0feba80d5a5ce74deb3
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Thu Sep 11 19:28:37 2025 +0100
[GLUTEN-6887][VL] Daily Update Velox Version (2025_09_11) (#10685)
* [GLUTEN-6887][VL] Daily Update Velox Version (2025_09_11)
Upstream Velox's New Commits:
630a4ea3c by Eric Jia, feat: Optimize type casting operations and instance
type checking in PlanNode.cpp (#14826)
345545a3d by Masha Basmanova, refactor: Remove unnecessary dependencies
from velox_cursor lib (#14828)
63f9644ea by Hongze Zhang, fix: Unused JSON field in
`TableWriteNode::create` (#14815)
afb250405 by Eric Jia, feat: Apply IndexLookupJoinBuilder to
indexlookupjoin test (#14823)
02552a666 by mbkkt, refactor: Use constants for special form names (#14734)
d80d2db87 by Patrick Sullivan, feat: Implement
geometry_to_dissolved_bing_tiles (#14251)
546e42929 by Xiaoxuan Meng, fix: Rename emptyUnnestValueName to markerName
and fix semantics (#14814)
3b6844be8 by Patrick Sullivan, feat: Implement geometry_to_bing_tiles
(#14253)
548787898 by Masha Basmanova, refactor: Remove unnecessary
registerConnectorFactory calls (#14818)
f4a2056ce by mbkkt, fix: Make access O(1) instead of O(n), it should reduce
overhead for large projections (#14403)
aad99903c by Eric Jia, feat: Add IndexLookupJoinBuilder into PlanBuilder
(#14813)
5db2697f1 by Masha Basmanova, refactor: Delete Connector::metadata() API
(#14809)
226b92cef by wecharyu, fix: Spark get_json_object supports bigint number
(#14545)
67480c240 by Masha Basmanova, refactor: Delete connector metadata
registries (#14806)
716f477e1 by Zhenyuan Zhao, fix: Make setType() support RowVector with
missing subfields (#14791)
94fbcae7e by Christian Zentgraf, fix(build): Replace actor with owner for
determining merge base (#14706)
71a46d93e by Natasha Sehgal, feat: Add P4HyperLogLog Type (#14626)
70c6392ee by Christian Zentgraf, fix(build): GCC13 compile errors (#14781)
b2f3dedd5 by PHILO-HE, docs(spark-functions): Clarify ANSI-compliance
(#14698)
dfb493654 by Sutou Kouhei, feat(build): Add support for CMake package
(#14738)
Signed-off-by: glutenperfbot <[email protected]>
* Fix unnest
---------
Signed-off-by: glutenperfbot <[email protected]>
Co-authored-by: glutenperfbot <[email protected]>
Co-authored-by: Rui Mo <[email protected]>
---
.../gluten/execution/GenerateExecTransformer.scala | 24 +++++++++++-----------
cpp/velox/substrait/SubstraitToVeloxPlan.cc | 6 +++---
ep/build-velox/src/get_velox.sh | 4 ++--
3 files changed, 17 insertions(+), 17 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
index e2d257c0c0..8cca495333 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
@@ -290,21 +290,21 @@ object PullOutGenerateProjectHelper extends
PullOutProjectHelper {
}
if (generate.outer) {
- val isNullOrEmpty =
+ val isPresent =
AttributeReference(generatePostAliasName, BooleanType, nullable
= true)()
val newOutput = (ordinal +: generate.generatorOutput.tail).map {
attr =>
val caseWhen = CaseWhen(
- Seq((isNullOrEmpty, Literal(null, attr.dataType))),
- attr
+ Seq((isPresent, attr)),
+ Literal(null, attr.dataType)
)
Alias(caseWhen, generatePostAliasName)(attr.exprId,
attr.qualifier)
}
// Reorder the generatorOutput to match the output from the Unnest
operator in Velox.
val newGenerate = generate.copy(generatorOutput =
- generate.generatorOutput.tail :+ originalOrdinal :+
isNullOrEmpty)
+ generate.generatorOutput.tail :+ originalOrdinal :+ isPresent)
ProjectExec(generate.requiredChildOutput ++ newOutput, newGenerate)
} else {
@@ -321,15 +321,15 @@ object PullOutGenerateProjectHelper extends
PullOutProjectHelper {
alias.toAttribute
}
if (generate.outer) {
- val isNullOrEmpty =
+ val isPresent =
AttributeReference(generatePostAliasName, BooleanType, nullable
= true)()
- val newGenerate = generate.copy(generatorOutput =
Seq(unnestOutput) :+ isNullOrEmpty)
+ val newGenerate = generate.copy(generatorOutput =
Seq(unnestOutput) :+ isPresent)
val newOutput = generate.generatorOutput.zipWithIndex.map {
case (attr, i) =>
val getStructField = GetStructField(unnestOutput, i,
Some(attr.name))
val caseWhen = CaseWhen(
- Seq((isNullOrEmpty, Literal(null, getStructField.dataType))),
- getStructField
+ Seq((isPresent, getStructField)),
+ Literal(null, getStructField.dataType)
)
Alias(caseWhen, generatePostAliasName)(attr.exprId,
attr.qualifier)
}
@@ -347,15 +347,15 @@ object PullOutGenerateProjectHelper extends
PullOutProjectHelper {
// Drop the last column of generatorOutput, which is the boolean
representing whether
// the null value is unnested from the input array/map (e.g.
array(1, null)), or the
// array/map itself is null or empty (e.g. array(), map(), null).
- val isNullOrEmpty =
+ val isPresent =
AttributeReference(generatePostAliasName, BooleanType, nullable =
true)()
val newGenerate =
- generate.copy(generatorOutput = generate.generatorOutput :+
isNullOrEmpty)
+ generate.copy(generatorOutput = generate.generatorOutput :+
isPresent)
val newOutput = generate.generatorOutput.map {
attr =>
val caseWhen = CaseWhen(
- Seq((isNullOrEmpty, Literal(null, attr.dataType))),
- attr
+ Seq((isPresent, attr)),
+ Literal(null, attr.dataType)
)
Alias(caseWhen, generatePostAliasName)(attr.exprId,
attr.qualifier)
}
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc
b/cpp/velox/substrait/SubstraitToVeloxPlan.cc
index 2df53f56e4..63dee7db8a 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc
@@ -858,18 +858,18 @@ core::PlanNodePtr
SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait::
}
std::optional<std::string> ordinalityName = std::nullopt;
- std::optional<std::string> emptyUnnestValueName = std::nullopt;
+ std::optional<std::string> markerName = std::nullopt;
if (generateRel.has_advanced_extension()) {
if
(SubstraitParser::configSetInOptimization(generateRel.advanced_extension(),
"isPosExplode=")) {
ordinalityName = std::make_optional<std::string>("pos");
}
if
(SubstraitParser::configSetInOptimization(generateRel.advanced_extension(),
"isOuter=")) {
- emptyUnnestValueName = std::make_optional<std::string>("empty_unnest");
+ markerName = std::make_optional<std::string>("marker");
}
}
return std::make_shared<core::UnnestNode>(
- nextPlanNodeId(), replicated, unnest, std::move(unnestNames),
ordinalityName, emptyUnnestValueName, childNode);
+ nextPlanNodeId(), replicated, unnest, std::move(unnestNames),
ordinalityName, markerName, childNode);
}
const core::WindowNode::Frame SubstraitToVeloxPlanConverter::createWindowFrame(
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index a218ca6254..ad741ba60a 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,11 +17,11 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2025_09_10
+VELOX_BRANCH=2025_09_11
VELOX_HOME=""
RUN_SETUP_SCRIPT=ON
VELOX_ENHANCED_REPO=https://github.com/IBM/velox.git
-VELOX_ENHANCED_BRANCH=ibm-2025_09_10
+VELOX_ENHANCED_BRANCH=ibm-2025_09_11
ENABLE_ENHANCED_FEATURES=OFF
# Developer use only for testing Velox PR.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
