This is an automated email from the ASF dual-hosted git repository.
wangzhen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 152be37c16 [CORE][UNIFFLE] Bump uniffle 0.9.1 (#8166)
152be37c16 is described below
commit 152be37c1669ef4b98e63e42b9653e23499ce633
Author: Zhen Wang <[email protected]>
AuthorDate: Mon Dec 9 14:08:34 2024 +0800
[CORE][UNIFFLE] Bump uniffle 0.9.1 (#8166)
* Bump uniffle 0.9.1
* address comment
* address comment
---
.github/workflows/velox_backend.yml | 23 ++++++++++-------------
docs/get-started/Velox.md | 2 +-
pom.xml | 2 +-
tools/gluten-it/pom.xml | 2 +-
4 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index ce195c4e1b..5feb5fa568 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -491,6 +491,8 @@ jobs:
fail-fast: false
matrix:
spark: [ "spark-3.2" ]
+ uniffle: [ "0.9.1" ]
+ hadoop: [ "2.8.5" ]
runs-on: ubuntu-20.04
container: centos:8
steps:
@@ -513,21 +515,16 @@ jobs:
run: |
yum update -y && yum install -y java-1.8.0-openjdk-devel wget git
$SETUP install_maven
- - name: Build for Uniffle 0.9.0
+ - name: Install Uniffle ${{ matrix.uniffle }}
run: |
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
cd /opt && \
- git clone -b v0.9.0 https://github.com/apache/incubator-uniffle.git
&& \
- cd incubator-uniffle && \
- $MVN_CMD clean install -Phadoop2.8,spark3 -DskipTests
- cd /opt && \
- ${WGET_CMD}
https://archive.apache.org/dist/incubator/uniffle/0.9.0/apache-uniffle-0.9.0-incubating-bin.tar.gz
&& \
- tar xzf apache-uniffle-0.9.0-incubating-bin.tar.gz -C /opt/ && mv
/opt/rss-0.9.0-hadoop2.8 /opt/uniffle && \
- ${WGET_CMD}
https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz
&& \
- tar xzf hadoop-2.8.5.tar.gz -C /opt/
- rm -rf /opt/incubator-uniffle
+ ${WGET_CMD} https://archive.apache.org/dist/incubator/uniffle/${{
matrix.uniffle }}/apache-uniffle-${{ matrix.uniffle }}-incubating-bin.tar.gz &&
\
+ mkdir /opt/uniffle && tar xzf apache-uniffle-${{ matrix.uniffle
}}-incubating-bin.tar.gz -C /opt/uniffle --strip-components=1 && \
+ ${WGET_CMD} https://archive.apache.org/dist/hadoop/common/hadoop-${{
matrix.hadoop }}/hadoop-${{ matrix.hadoop }}.tar.gz && \
+ tar xzf hadoop-${{ matrix.hadoop }}.tar.gz -C /opt/ && \
cd /opt/uniffle && mkdir shuffle_data && \
- bash -c "echo -e 'XMX_SIZE=16g\nHADOOP_HOME=/opt/hadoop-2.8.5' >
./bin/rss-env.sh" && \
+ bash -c "echo -e 'XMX_SIZE=16g\nHADOOP_HOME=/opt/hadoop-${{
matrix.hadoop }}' > ./bin/rss-env.sh" && \
bash -c "echo -e 'rss.coordinator.shuffle.nodes.max
1\nrss.rpc.server.port 19999' > ./conf/coordinator.conf" && \
bash -c "echo -e 'rss.server.app.expired.withoutHeartbeat
7200000\nrss.server.heartbeat.delay 3000\nrss.rpc.server.port
19997\nrss.rpc.server.type GRPC_NETTY\nrss.jetty.http.port
19996\nrss.server.netty.port 19995\nrss.storage.basePath
/opt/uniffle/shuffle_data\nrss.storage.type
MEMORY_LOCALFILE\nrss.coordinator.quorum
localhost:19999\nrss.server.flush.thread.alive
10\nrss.server.single.buffer.flush.threshold 64m' > ./conf/server.conf" && \
bash ./bin/start-coordinator.sh && bash ./bin/start-shuffle-server.sh
@@ -535,11 +532,11 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/ && \
$MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox
-Puniffle -DskipTests
- - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with
uniffle-0.9.0
+ - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with
uniffle-${{ matrix.uniffle }}
run: |
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
cd $GITHUB_WORKSPACE/tools/gluten-it && \
- $MVN_CMD clean install -Pspark-3.2 -Puniffle && \
+ $MVN_CMD clean install -P${{ matrix.spark }} -Puniffle && \
GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
--local --preset=velox-with-uniffle --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1
diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
index 3b3ab07a8d..a0af0cc75d 100644
--- a/docs/get-started/Velox.md
+++ b/docs/get-started/Velox.md
@@ -272,7 +272,7 @@ spark.dynamicAllocation.enabled false
## Uniffle support
-Uniffle with velox backend supports
[Uniffle](https://github.com/apache/incubator-uniffle) as remote shuffle
service. Currently, the supported Uniffle versions are `0.9.0`.
+Uniffle with velox backend supports
[Uniffle](https://github.com/apache/incubator-uniffle) as remote shuffle
service. Currently, the supported Uniffle versions are `0.9.1`.
First refer to this URL(https://uniffle.apache.org/docs/intro) to get start
with uniffle.
diff --git a/pom.xml b/pom.xml
index a2cccda1e4..9b27a68397 100644
--- a/pom.xml
+++ b/pom.xml
@@ -66,7 +66,7 @@
<delta.version>2.4.0</delta.version>
<delta.binary.version>24</delta.binary.version>
<celeborn.version>0.5.2</celeborn.version>
- <uniffle.version>0.9.0</uniffle.version>
+ <uniffle.version>0.9.1</uniffle.version>
<arrow.version>15.0.0</arrow.version>
<arrow-gluten.version>15.0.0-gluten</arrow-gluten.version>
<arrow-memory.artifact>arrow-memory-unsafe</arrow-memory.artifact>
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index 1973a7d998..eb2c3d87e7 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -22,7 +22,7 @@
<scala.binary.version>2.12</scala.binary.version>
<spark.major.version>3</spark.major.version>
<celeborn.version>0.3.2-incubating</celeborn.version>
- <uniffle.version>0.9.0</uniffle.version>
+ <uniffle.version>0.9.1</uniffle.version>
<gluten.version>1.3.0-SNAPSHOT</gluten.version>
<guava.version>32.0.1-jre</guava.version>
<tpch.version>1.1</tpch.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
