This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3ff9254d9c [GLUTEN-6887][VL] Daily Update Velox Version (2025_11_20)
(#11129)
3ff9254d9c is described below
commit 3ff9254d9cf72e1e0fa58942a4b3d2fd8dc7cb54
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Thu Nov 20 21:06:40 2025 +0000
[GLUTEN-6887][VL] Daily Update Velox Version (2025_11_20) (#11129)
* [GLUTEN-6887][VL] Daily Update Velox Version (dft-2025_11_20)
Upstream Velox's New Commits:
1944aab79 by Jake Jung, refactor(exception): Decouple line number logging
logic (#15521)
7eab76d4a by Masha Basmanova, refactor: Move toSubfieldFilter API to
PlanBuilder (#15556)
ecddafb76 by Masha Basmanova, refactor: Change leafCallToSubfieldFilter API
to return an optional pair (#15555)
f35d24c49 by Bikramjeet Vig, fix: Max length estimation in
DateTimeFormatter (#15532)
Signed-off-by: glutenperfbot <[email protected]>
* Update SparkExprToSubfieldFilterParser.h
* Remove toSubfieldFilter implementation
Removed the implementation of toSubfieldFilter function, which handled
logical 'or' and 'not' expressions.
* Update SparkExprToSubfieldFilterParser.h
* Refactor leafCallToSubfieldFilter to use std::nullopt
* Update types in leafCallToSubfieldFilter method
* Update combine function to use facebook::velox namespace
* fix compile, limit cache size
Signed-off-by: Yuan <[email protected]>
---------
Signed-off-by: glutenperfbot <[email protected]>
Signed-off-by: Yuan <[email protected]>
Co-authored-by: glutenperfbot <[email protected]>
Co-authored-by: BInwei Yang <[email protected]>
Co-authored-by: Yuan <[email protected]>
---
.github/workflows/velox_backend_cache.yml | 5 ++
.github/workflows/velox_backend_x86.yml | 2 +
.../functions/SparkExprToSubfieldFilterParser.cc | 76 ++++++++++------------
.../functions/SparkExprToSubfieldFilterParser.h | 8 +--
dev/ci-velox-buildstatic-centos-7.sh | 1 -
ep/build-velox/src/get-velox.sh | 4 +-
6 files changed, 46 insertions(+), 50 deletions(-)
diff --git a/.github/workflows/velox_backend_cache.yml
b/.github/workflows/velox_backend_cache.yml
index db17eadf9c..60ca54ecda 100644
--- a/.github/workflows/velox_backend_cache.yml
+++ b/.github/workflows/velox_backend_cache.yml
@@ -45,6 +45,7 @@ jobs:
run: |
docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7 bash -c "
export CCACHE_DIR=/work/.ccache
+ export CCACHE_MAXSIZE=1G
mkdir -p /work/.ccache
bash dev/ci-velox-buildstatic-centos-7.sh
"
@@ -73,6 +74,7 @@ jobs:
ccache-centos8-release-default-${{runner.arch}}
- name: Build Gluten native libraries
run: |
+ export CCACHE_MAXSIZE=1G
bash dev/ci-velox-buildstatic-centos-9.sh
- name: Save Ccache
uses: actions/cache/save@v3
@@ -100,6 +102,7 @@ jobs:
- name: Build Gluten shared libraries
run: |
df -a
+ export CCACHE_MAXSIZE=1G
bash dev/ci-velox-buildshared-centos-8.sh
ccache -s
- name: Save Ccache
@@ -125,6 +128,7 @@ jobs:
run: |
docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7 bash -c "
export CCACHE_DIR=/work/.ccache
+ export CCACHE_MAXSIZE=1G
mkdir -p /work/.ccache
bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
"
@@ -154,6 +158,7 @@ jobs:
- name: Build Gluten shared libraries
run: |
df -a
+ export CCACHE_MAXSIZE=1G
bash dev/buildbundle-veloxbe.sh --run_setup_script=OFF
--build_arrow=OFF --spark_version=3.4 --enable_gpu=ON
ccache -s
- name: Save Ccache
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index e87b822b3d..c200b79a62 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -73,7 +73,9 @@ jobs:
df -a
cd /work
export CCACHE_DIR=/work/.ccache
+ export CCACHE_MAXSIZE=1G
mkdir -p /work/.ccache
+ ccache -sz
bash dev/ci-velox-buildstatic-centos-7.sh
ccache -s
mkdir -p /work/.m2/repository/org/apache/arrow/
diff --git a/cpp/velox/operators/functions/SparkExprToSubfieldFilterParser.cc
b/cpp/velox/operators/functions/SparkExprToSubfieldFilterParser.cc
index c1bc452bd8..20baaa413e 100644
--- a/cpp/velox/operators/functions/SparkExprToSubfieldFilterParser.cc
+++ b/cpp/velox/operators/functions/SparkExprToSubfieldFilterParser.cc
@@ -20,87 +20,81 @@ namespace gluten {
using namespace facebook::velox;
-std::pair<common::Subfield, std::unique_ptr<common::Filter>>
SparkExprToSubfieldFilterParser::toSubfieldFilter(
- const core::TypedExprPtr& expr,
- core::ExpressionEvaluator* evaluator) {
- if (expr->isCallKind(); auto* call =
expr->asUnchecked<core::CallTypedExpr>()) {
- if (call->name() == "or") {
- auto left = toSubfieldFilter(call->inputs()[0], evaluator);
- auto right = toSubfieldFilter(call->inputs()[1], evaluator);
- VELOX_CHECK(left.first == right.first);
- return {std::move(left.first), makeOrFilter(std::move(left.second),
std::move(right.second))};
- }
- common::Subfield subfield;
- std::unique_ptr<common::Filter> filter;
- if (call->name() == "not") {
- if (auto* inner = call->inputs()[0]->asUnchecked<core::CallTypedExpr>())
{
- filter = leafCallToSubfieldFilter(*inner, subfield, evaluator, true);
- }
- } else {
- filter = leafCallToSubfieldFilter(*call, subfield, evaluator, false);
- }
- if (filter) {
- return std::make_pair(std::move(subfield), std::move(filter));
- }
+namespace {
+std::optional<std::pair<facebook::velox::common::Subfield,
std::unique_ptr<facebook::velox::common::Filter>>> combine(
+ facebook::velox::common::Subfield& subfield,
+ std::unique_ptr<facebook::velox::common::Filter>& filter) {
+ if (filter != nullptr) {
+ return std::make_pair(std::move(subfield), std::move(filter));
}
- VELOX_UNSUPPORTED("Unsupported expression for range filter: {}",
expr->toString());
+
+ return std::nullopt;
}
+} // namespace
-std::unique_ptr<common::Filter>
SparkExprToSubfieldFilterParser::leafCallToSubfieldFilter(
+std::optional<std::pair<facebook::velox::common::Subfield,
std::unique_ptr<facebook::velox::common::Filter>>>
+SparkExprToSubfieldFilterParser::leafCallToSubfieldFilter(
const core::CallTypedExpr& call,
- common::Subfield& subfield,
core::ExpressionEvaluator* evaluator,
bool negated) {
if (call.inputs().empty()) {
- return nullptr;
+ return std::nullopt;
}
const auto* leftSide = call.inputs()[0].get();
+ common::Subfield subfield;
if (call.name() == "equalto") {
if (toSubfield(leftSide, subfield)) {
- return negated ? makeNotEqualFilter(call.inputs()[1], evaluator) :
makeEqualFilter(call.inputs()[1], evaluator);
+ auto filter =
+ negated ? makeNotEqualFilter(call.inputs()[1], evaluator) :
makeEqualFilter(call.inputs()[1], evaluator);
+ return combine(subfield, filter);
}
} else if (call.name() == "lessthanorequal") {
if (toSubfield(leftSide, subfield)) {
- return negated ? makeGreaterThanFilter(call.inputs()[1], evaluator)
- : makeLessThanOrEqualFilter(call.inputs()[1], evaluator);
+ auto filter = negated ? makeGreaterThanFilter(call.inputs()[1],
evaluator)
+ : makeLessThanOrEqualFilter(call.inputs()[1],
evaluator);
+ return combine(subfield, filter);
}
} else if (call.name() == "lessthan") {
if (toSubfield(leftSide, subfield)) {
- return negated ? makeGreaterThanOrEqualFilter(call.inputs()[1],
evaluator)
- : makeLessThanFilter(call.inputs()[1], evaluator);
+ auto filter = negated ? makeGreaterThanOrEqualFilter(call.inputs()[1],
evaluator)
+ : makeLessThanFilter(call.inputs()[1], evaluator);
+ return combine(subfield, filter);
}
} else if (call.name() == "greaterthanorequal") {
if (toSubfield(leftSide, subfield)) {
- return negated ? makeLessThanFilter(call.inputs()[1], evaluator)
- : makeGreaterThanOrEqualFilter(call.inputs()[1],
evaluator);
+ auto filter = negated ? makeLessThanFilter(call.inputs()[1], evaluator)
+ : makeGreaterThanOrEqualFilter(call.inputs()[1],
evaluator);
+ return combine(subfield, filter);
}
} else if (call.name() == "greaterthan") {
if (toSubfield(leftSide, subfield)) {
- return negated ? makeLessThanOrEqualFilter(call.inputs()[1], evaluator)
- : makeGreaterThanFilter(call.inputs()[1], evaluator);
+ auto filter = negated ? makeLessThanOrEqualFilter(call.inputs()[1],
evaluator)
+ : makeGreaterThanFilter(call.inputs()[1],
evaluator);
+ return combine(subfield, filter);
}
} else if (call.name() == "in") {
if (toSubfield(leftSide, subfield)) {
- return makeInFilter(call.inputs()[1], evaluator, negated);
+ auto filter = makeInFilter(call.inputs()[1], evaluator, negated);
+ return combine(subfield, filter);
}
} else if (call.name() == "isnull") {
if (toSubfield(leftSide, subfield)) {
if (negated) {
- return exec::isNotNull();
+ return std::make_pair(std::move(subfield),
facebook::velox::exec::isNotNull());
}
- return exec::isNull();
+ return std::make_pair(std::move(subfield),
facebook::velox::exec::isNull());
}
} else if (call.name() == "isnotnull") {
if (toSubfield(leftSide, subfield)) {
if (negated) {
- return exec::isNull();
+ return std::make_pair(std::move(subfield),
facebook::velox::exec::isNull());
}
- return exec::isNotNull();
+ return std::make_pair(std::move(subfield),
facebook::velox::exec::isNotNull());
}
}
- return nullptr;
+ return std::nullopt;
}
} // namespace gluten
diff --git a/cpp/velox/operators/functions/SparkExprToSubfieldFilterParser.h
b/cpp/velox/operators/functions/SparkExprToSubfieldFilterParser.h
index 4b699fb172..0f7d826446 100644
--- a/cpp/velox/operators/functions/SparkExprToSubfieldFilterParser.h
+++ b/cpp/velox/operators/functions/SparkExprToSubfieldFilterParser.h
@@ -23,13 +23,9 @@ namespace gluten {
/// 2) The supported functions vary.
class SparkExprToSubfieldFilterParser : public
facebook::velox::exec::ExprToSubfieldFilterParser {
public:
- std::pair<facebook::velox::common::Subfield,
std::unique_ptr<facebook::velox::common::Filter>> toSubfieldFilter(
- const facebook::velox::core::TypedExprPtr& expr,
- facebook::velox::core::ExpressionEvaluator* evaluator) override;
-
- std::unique_ptr<facebook::velox::common::Filter> leafCallToSubfieldFilter(
+ std::optional<std::pair<facebook::velox::common::Subfield,
std::unique_ptr<facebook::velox::common::Filter>>>
+ leafCallToSubfieldFilter(
const facebook::velox::core::CallTypedExpr& call,
- facebook::velox::common::Subfield& subfield,
facebook::velox::core::ExpressionEvaluator* evaluator,
bool negated) override;
};
diff --git a/dev/ci-velox-buildstatic-centos-7.sh
b/dev/ci-velox-buildstatic-centos-7.sh
index 6737c6329c..29b707f035 100755
--- a/dev/ci-velox-buildstatic-centos-7.sh
+++ b/dev/ci-velox-buildstatic-centos-7.sh
@@ -19,6 +19,5 @@ set -e
source /opt/rh/devtoolset-11/enable
source /opt/rh/rh-git227/enable
-export NUM_THREADS=2
./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF
--build_tests=OFF --build_benchmarks=OFF \
--build_examples=OFF --enable_s3=ON --enable_gcs=ON
--enable_hdfs=ON --enable_abfs=ON
diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh
index daed97b219..b71568bee5 100755
--- a/ep/build-velox/src/get-velox.sh
+++ b/ep/build-velox/src/get-velox.sh
@@ -18,8 +18,8 @@ set -exu
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
VELOX_REPO=https://github.com/IBM/velox.git
-VELOX_BRANCH=dft-2025_11_19
-VELOX_ENHANCED_BRANCH=ibm-2025_11_19
+VELOX_BRANCH=dft-2025_11_20
+VELOX_ENHANCED_BRANCH=ibm-2025_11_20
VELOX_HOME=""
RUN_SETUP_SCRIPT=ON
ENABLE_ENHANCED_FEATURES=OFF
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
