This is an automated email from the ASF dual-hosted git repository.
kerwinzhang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f8e4c36d07 [GLUTEN-10675][CELEBORN] Bump Celeborn version to 0.6.1
(#10677)
f8e4c36d07 is described below
commit f8e4c36d0738082ff7b76a2e9f0d4ef47b98650b
Author: Nicholas Jiang <[email protected]>
AuthorDate: Thu Sep 11 19:19:58 2025 +0800
[GLUTEN-10675][CELEBORN] Bump Celeborn version to 0.6.1 (#10677)
---
.github/workflows/velox_backend_x86.yml | 4 ++--
dev/docker/Dockerfile.centos8-dynamic-build | 2 +-
dev/docker/Dockerfile.centos9-dynamic-build | 2 +-
.../shuffle/gluten/celeborn/CelebornShuffleManager.java | 3 ++-
.../spark/shuffle/gluten/celeborn/CelebornUtils.java | 14 ++++++++++----
pom.xml | 2 +-
tools/gluten-it/pom.xml | 4 ++--
7 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 7cd753fb56..9fbc5596e9 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -558,7 +558,7 @@ jobs:
fail-fast: false
matrix:
spark: [ "spark-3.2" ]
- celeborn: [ "celeborn-0.6.0", "celeborn-0.5.4", "celeborn-0.4.3"]
+ celeborn: [ "celeborn-0.6.1", "celeborn-0.5.4", "celeborn-0.4.3"]
runs-on: ubuntu-22.04
container: apache/gluten:centos-8-jdk8
steps:
@@ -584,7 +584,7 @@ jobs:
EXTRA_PROFILE="-Pceleborn-0.4"
elif [ "${{ matrix.celeborn }}" = "celeborn-0.5.4" ]; then
EXTRA_PROFILE="-Pceleborn-0.5"
- elif [ "${{ matrix.celeborn }}" = "celeborn-0.6.0" ]; then
+ elif [ "${{ matrix.celeborn }}" = "celeborn-0.6.1" ]; then
EXTRA_PROFILE="-Pceleborn-0.6"
fi
echo "EXTRA_PROFILE: ${EXTRA_PROFILE}"
diff --git a/dev/docker/Dockerfile.centos8-dynamic-build
b/dev/docker/Dockerfile.centos8-dynamic-build
index 04d5da8571..29053fdc1a 100644
--- a/dev/docker/Dockerfile.centos8-dynamic-build
+++ b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -42,7 +42,7 @@ RUN set -ex; \
rm -rf ${local_binary}; \
wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.4.3/apache-celeborn-0.4.3-bin.tgz
-P /opt/; \
wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz
-P /opt/; \
- wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.6.0/apache-celeborn-0.6.0-bin.tgz
-P /opt/; \
+ wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz
-P /opt/; \
wget -nv
https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz
-P /opt/; \
wget -nv
https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz
-P /opt/; \
git clone --depth=1 https://github.com/apache/incubator-gluten
/opt/gluten; \
diff --git a/dev/docker/Dockerfile.centos9-dynamic-build
b/dev/docker/Dockerfile.centos9-dynamic-build
index e50268cbc0..7fffe6cd3d 100644
--- a/dev/docker/Dockerfile.centos9-dynamic-build
+++ b/dev/docker/Dockerfile.centos9-dynamic-build
@@ -40,7 +40,7 @@ RUN set -ex; \
rm -rf ${local_binary}; \
wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.4.3/apache-celeborn-0.4.3-bin.tgz
-P /opt/; \
wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz
-P /opt/; \
- wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.6.0/apache-celeborn-0.6.0-bin.tgz
-P /opt/; \
+ wget -nv
https://archive.apache.org/dist/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz
-P /opt/; \
wget -nv
https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz
-P /opt/; \
wget -nv
https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz
-P /opt/; \
git clone --depth=1 https://github.com/apache/incubator-gluten
/opt/gluten; \
diff --git
a/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
b/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
index 9647095e8b..31d82ce0c7 100644
---
a/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
+++
b/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
@@ -281,7 +281,8 @@ public class CelebornShuffleManager
_vanillaCelebornShuffleManager = null;
}
if (failedShuffleCleaner != null) {
- CelebornUtils.resetFailedShuffleCleaner(failedShuffleCleaner);
+ CelebornUtils.stopFailedShuffleCleaner(failedShuffleCleaner);
+ failedShuffleCleaner = null;
}
}
diff --git
a/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
b/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
index d28726ac3c..e8a8fcca60 100644
---
a/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
+++
b/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
@@ -684,11 +684,17 @@ public class CelebornUtils {
}
}
- public static void resetFailedShuffleCleaner(Object failedShuffleCleaner) {
+ public static void stopFailedShuffleCleaner(Object failedShuffleCleaner) {
try {
- // for Celeborn 0.6.0
- Method resetMethod =
failedShuffleCleaner.getClass().getDeclaredMethod("reset");
- resetMethod.invoke(failedShuffleCleaner);
+ try {
+ // for Celeborn 0.6.1
+ Method stopMethod =
failedShuffleCleaner.getClass().getDeclaredMethod("stop");
+ stopMethod.invoke(failedShuffleCleaner);
+ } catch (NoSuchMethodException e) {
+ // for Celeborn 0.6.0
+ Method resetMethod =
failedShuffleCleaner.getClass().getDeclaredMethod("reset");
+ resetMethod.invoke(failedShuffleCleaner);
+ }
} catch (NoSuchMethodException ignored) {
} catch (Exception e) {
throw new RuntimeException(e);
diff --git a/pom.xml b/pom.xml
index a7f1406d76..af074cf931 100644
--- a/pom.xml
+++ b/pom.xml
@@ -80,7 +80,7 @@
<delta.package.name>delta-core</delta.package.name>
<delta.version>2.4.0</delta.version>
<delta.binary.version>24</delta.binary.version>
- <celeborn.version>0.6.0</celeborn.version>
+ <celeborn.version>0.6.1</celeborn.version>
<uniffle.version>0.9.2</uniffle.version>
<arrow.version>15.0.0</arrow.version>
<arrow-gluten.version>15.0.0-gluten</arrow-gluten.version>
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index 15d772282c..f70de42e9a 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -22,7 +22,7 @@
<spark.major.version>3</spark.major.version>
<delta.package.name>delta-core</delta.package.name>
<delta.version>2.4.0</delta.version>
- <celeborn.version>0.6.0</celeborn.version>
+ <celeborn.version>0.6.1</celeborn.version>
<uniffle.version>0.9.2</uniffle.version>
<gluten.version>1.6.0-SNAPSHOT</gluten.version>
<tpch.version>1.1</tpch.version>
@@ -306,7 +306,7 @@
<profile>
<id>celeborn-0.6</id>
<properties>
- <celeborn.version>0.6.0</celeborn.version>
+ <celeborn.version>0.6.1</celeborn.version>
</properties>
</profile>
</profiles>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
