This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 82d97bb15e [GLUTEN-6887][VL] Daily Update Velox Version (2025_11_22)
(#11152)
82d97bb15e is described below
commit 82d97bb15e8fb002f44b65389e8081f260d0723b
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Mon Nov 24 14:12:59 2025 +0000
[GLUTEN-6887][VL] Daily Update Velox Version (2025_11_22) (#11152)
* [GLUTEN-6887][VL] Daily Update Velox Version (dft-2025_11_22)
Upstream Velox's New Commits:
fb8d35644 by Mingliang Zhu, feat: Support int64_t value for Spark
timestampadd function (#15459)
9bc0ce3b5 by Wei He, misc(fuzzer): Skip $internal$split_to_map in
expression fuzzer (#15596)
235cb8238 by Devavret Makkar, feat: Add a CudfTpchBenchmark (#13695)
e11eb4bc9 by Zac Wen, refactor: Name magic numbers in cache (#15553)
Signed-off-by: glutenperfbot <[email protected]>
* Exclude additional tests for timestampadd issues
* free disk space for celeborn/uniffle, fix filename
Signed-off-by: Yuan <[email protected]>
* always save ccache
Signed-off-by: Yuan <[email protected]>
---------
Signed-off-by: glutenperfbot <[email protected]>
Signed-off-by: Yuan <[email protected]>
Co-authored-by: glutenperfbot <[email protected]>
Co-authored-by: BInwei Yang <[email protected]>
Co-authored-by: Yuan <[email protected]>
---
.github/workflows/velox_backend_x86.yml | 5 +++--
dev/ci-velox-buildstatic-centos-7.sh | 1 +
dev/ci-velox-buildstatic-centos-8.sh | 1 -
dev/docker/Dockerfile.centos8-dynamic-build | 8 ++++----
dev/docker/Dockerfile.centos9-dynamic-build | 8 ++++----
ep/build-velox/src/get-velox.sh | 4 ++--
.../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 4 ++++
7 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index c99d936732..e8a1a6e9ac 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -83,6 +83,7 @@ jobs:
"
- name: "Save ccache"
+ if: always()
uses: actions/cache/save@v4
id: ccache
with:
@@ -531,7 +532,7 @@ jobs:
if [ ! -e "/opt/hadoop-${{ matrix.hadoop }}.tar.gz" ]; then
${WGET_CMD}
https://archive.apache.org/dist/hadoop/common/hadoop-${{ matrix.hadoop
}}/hadoop-${{ matrix.hadoop }}.tar.gz -P /opt
fi
- cd /opt && \
+ cd /opt && rm -rf shims && \
mkdir /opt/uniffle && tar xzf apache-uniffle-${{ matrix.uniffle
}}-incubating-bin.tar.gz -C /opt/uniffle --strip-components=1 && \
tar xzf hadoop-${{ matrix.hadoop }}.tar.gz -C /opt/ && \
cd /opt/uniffle && mkdir shuffle_data && \
@@ -590,7 +591,7 @@ jobs:
echo "WARNING: please pre-install your required package in docker
image since the downloading is throttled by this site."
wget -nv https://archive.apache.org/dist/celeborn/${{
matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz -P /opt/
fi
- cd /opt && mkdir -p celeborn && \
+ cd /opt && rm -rf shims && mkdir -p celeborn && \
tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn
--strip-components=1 && cd celeborn && \
mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
bash -c "echo -e
'CELEBORN_MASTER_MEMORY=8g\nCELEBORN_WORKER_MEMORY=8g\nCELEBORN_WORKER_OFFHEAP_MEMORY=16g'
> ./conf/celeborn-env.sh" && \
diff --git a/dev/ci-velox-buildstatic-centos-7.sh
b/dev/ci-velox-buildstatic-centos-7.sh
index 29b707f035..8d99200500 100755
--- a/dev/ci-velox-buildstatic-centos-7.sh
+++ b/dev/ci-velox-buildstatic-centos-7.sh
@@ -19,5 +19,6 @@ set -e
source /opt/rh/devtoolset-11/enable
source /opt/rh/rh-git227/enable
+export NUM_THREADS=4
./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF
--build_tests=OFF --build_benchmarks=OFF \
--build_examples=OFF --enable_s3=ON --enable_gcs=ON
--enable_hdfs=ON --enable_abfs=ON
diff --git a/dev/ci-velox-buildstatic-centos-8.sh
b/dev/ci-velox-buildstatic-centos-8.sh
index 0bf7f56601..e3f45300d6 100644
--- a/dev/ci-velox-buildstatic-centos-8.sh
+++ b/dev/ci-velox-buildstatic-centos-8.sh
@@ -18,7 +18,6 @@
set -e
source /opt/rh/gcc-toolset-11/enable
-export NUM_THREADS=2
if [ "$(uname -m)" = "aarch64" ]; then
export CPU_TARGET="aarch64";
export VCPKG_FORCE_SYSTEM_BINARIES=1;
diff --git a/dev/docker/Dockerfile.centos8-dynamic-build
b/dev/docker/Dockerfile.centos8-dynamic-build
index 65f8b9dd64..2d7ab9c9df 100644
--- a/dev/docker/Dockerfile.centos8-dynamic-build
+++ b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -40,10 +40,10 @@ RUN set -ex; \
tar -xvf ${local_binary}; \
mv apache-maven-${maven_version} /usr/lib/maven; \
rm -rf ${local_binary}; \
- wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-P /opt/; \
- wget -nv
${mirror_host}/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz?action=download
-P /opt/; \
- wget -nv
${mirror_host}/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz?action=download
-P /opt/; \
- wget -nv ${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P
/opt/; \
+ wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
+ wget -nv
${mirror_host}/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.1-bin.tgz; \
+ wget -nv
${mirror_host}/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz?action=download
-O /opt/apache-uniffle-0.9.2-incubating-bin.tar.gz; \
+ wget -nv
${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz?action=download
-O /opt/hadoop-2.8.5.tar.gz; \
git clone --depth=1 https://github.com/apache/incubator-gluten
/opt/gluten; \
cd /opt/gluten/.github/workflows/util/; \
./install-spark-resources.sh 3.2; \
diff --git a/dev/docker/Dockerfile.centos9-dynamic-build
b/dev/docker/Dockerfile.centos9-dynamic-build
index 7117e6e530..d1eae3e204 100644
--- a/dev/docker/Dockerfile.centos9-dynamic-build
+++ b/dev/docker/Dockerfile.centos9-dynamic-build
@@ -38,10 +38,10 @@ RUN set -ex; \
tar -xvf ${local_binary}; \
mv apache-maven-${maven_version} /usr/lib/maven; \
rm -rf ${local_binary}; \
- wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-P /opt/; \
- wget -nv
${mirror_host}/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz?action=download
-P /opt/; \
- wget -nv
${mirror_host}/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz?action=download
-P /opt/; \
- wget -nv ${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P
/opt/; \
+ wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
+ wget -nv
${mirror_host}/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.1-bin.tgz; \
+ wget -nv
${mirror_host}/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz?action=download
-O /opt/apache-uniffle-0.9.2-incubating-bin.tar.gz; \
+ wget -nv
${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz?action=download
-O /opt/hadoop-2.8.5.tar.gz; \
git clone --depth=1 https://github.com/apache/incubator-gluten
/opt/gluten; \
cd /opt/gluten/.github/workflows/util/; \
./install-spark-resources.sh 3.2; \
diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh
index ed39a4ff5c..b713519bb3 100755
--- a/ep/build-velox/src/get-velox.sh
+++ b/ep/build-velox/src/get-velox.sh
@@ -18,8 +18,8 @@ set -exu
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
VELOX_REPO=https://github.com/IBM/velox.git
-VELOX_BRANCH=dft-2025_11_21
-VELOX_ENHANCED_BRANCH=ibm-2025_11_21
+VELOX_BRANCH=dft-2025_11_22
+VELOX_ENHANCED_BRANCH=ibm-2025_11_22
VELOX_HOME=""
RUN_SETUP_SCRIPT=ON
ENABLE_ENHANCED_FEATURES=OFF
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index c1ffb63301..4c66d77e57 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -148,6 +148,10 @@ class VeloxTestSettings extends BackendTestSettings {
// Vanilla Spark does not have a unified DST Timestamp fastTime.
1320570000000L and
// 1320566400000L both represent 2011-11-06 01:00:00.
.exclude("SPARK-42635: timestampadd near daylight saving transition")
+
+ // TODO: fix in Spark-4.0
+ .exclude("SPARK-50669: timestampadd with long types")
+
//
https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145
.excludeGlutenTest("from_unixtime")
// Replaced by a gluten test to pass timezone through config.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
