This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6b9d97c2a [GLUTEN-4917][VL] Enable celeborn test in CI (#5247)
6b9d97c2a is described below
commit 6b9d97c2a1b2897c69e99e17530c9d244829a688
Author: PHILO-HE <[email protected]>
AuthorDate: Wed Apr 3 16:29:39 2024 +0800
[GLUTEN-4917][VL] Enable celeborn test in CI (#5247)
---
.github/workflows/velox_docker.yml | 44 ++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/.github/workflows/velox_docker.yml
b/.github/workflows/velox_docker.yml
index 793abf78e..ddee9b496 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -275,6 +275,50 @@ jobs:
# -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
# -d=OFFHEAP_SIZE:1g,spark.memory.offHeap.size=1g || true
+ run-tpc-test-ubuntu-2204-celeborn:
+ needs: build-native-lib
+ strategy:
+ fail-fast: false
+ matrix:
+ spark: ["spark-3.2"]
+ celeborn: ["celeborn-0.4.0", "celeborn-0.3.2"]
+ runs-on: ubuntu-20.04
+ container: ubuntu:22.04
+ steps:
+ - uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Setup java and maven
+ run: |
+ apt-get update && apt-get install -y openjdk-8-jdk maven wget
+ - name: Build for Spark ${{ matrix.spark }}
+ run: |
+ cd $GITHUB_WORKSPACE/
+ mvn clean install -P${{ matrix.spark }} -Pbackends-velox,rss
-DskipTests
+ - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{
matrix.celeborn }}
+ run: |
+ export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ EXTRA_PROFILE=""
+ if [ "${{ matrix.celeborn }}" = "celeborn-0.4.0" ]; then
+ EXTRA_PROFILE="-Pceleborn-0.4"
+ fi
+ echo "EXTRA_PROFILE: ${EXTRA_PROFILE}"
+ cd /opt && mkdir -p celeborn && \
+ wget https://archive.apache.org/dist/incubator/celeborn/${{
matrix.celeborn }}-incubating/apache-${{ matrix.celeborn }}-incubating-bin.tgz
&& \
+ tar xzf apache-${{ matrix.celeborn }}-incubating-bin.tgz -C
/opt/celeborn --strip-components=1 && cd celeborn && \
+ mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
+ bash -c "echo -e
'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g'
> ./conf/celeborn-env.sh" && \
+ bash -c "echo -e 'celeborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf"
&& \
+ bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \
+ cd $GITHUB_WORKSPACE/tools/gluten-it && mvn clean install
-Pspark-3.2,rss ${EXTRA_PROFILE} && \
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox-with-celeborn --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=8 --iterations=1 && \
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox-with-celeborn --benchmark-type=ds
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=8 --iterations=1
+
run-spark-test-spark32:
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
