This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3acf27279 [GLUTEN-4917][VL][CI] Enable Gluten CPP tests (#5114)
3acf27279 is described below
commit 3acf272798544350647d0ace9a3a97d11f00b3ad
Author: PHILO-HE <[email protected]>
AuthorDate: Tue Apr 2 11:07:24 2024 +0800
[GLUTEN-4917][VL][CI] Enable Gluten CPP tests (#5114)
---
.github/workflows/velox_docker.yml | 85 ++++++--------------------------------
1 file changed, 12 insertions(+), 73 deletions(-)
diff --git a/.github/workflows/velox_docker.yml
b/.github/workflows/velox_docker.yml
index 40a8d4fff..597fca081 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -47,7 +47,7 @@ concurrency:
jobs:
build-native-lib:
runs-on: ubuntu-20.04
- container: inteldpo/gluten-centos-packaging:latest # centos7 with
depedencies installed
+ container: inteldpo/gluten-centos-packaging:latest # centos7 with
dependencies installed
steps:
- uses: actions/checkout@v2
- name: Generate cache key
@@ -70,7 +70,7 @@ jobs:
cd $GITHUB_WORKSPACE/ && \
sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt &&
\
export NUM_THREADS=4
- ./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF
--enable_s3=ON \
+ ./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF
--enable_s3=ON \
--enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
- uses: actions/upload-artifact@v2
with:
@@ -284,74 +284,6 @@ jobs:
# -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
# -d=OFFHEAP_SIZE:1g,spark.memory.offHeap.size=1g || true
- # run-tpc-test-centos8-celeborn:
- # needs: build-native-lib
- # strategy:
- # fail-fast: false
- # matrix:
- # spark: ["spark-3.2"]
- # runs-on: ubuntu-20.04
- # container: centos:8
- # steps:
- # - uses: actions/checkout@v2
- # - name: Download All Artifacts
- # uses: actions/download-artifact@v2
- # with:
- # name: velox-native-lib-${{github.sha}}
- # path: ./cpp/build/releases
- # - name: Update mirror list
- # run: |
- # sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- # sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- # - name: Setup java and maven
- # run: |
- # yum update -y && yum install -y java-1.8.0-openjdk-devel wget
- # wget
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- # tar -xvf apache-maven-3.8.8-bin.tar.gz
- # mv apache-maven-3.8.8 /usr/lib/maven
- # - name: Build for Spark ${{ matrix.spark }}
- # run: |
- # cd $GITHUB_WORKSPACE/
- # export MAVEN_HOME=/usr/lib/maven
- # export PATH=${PATH}:${MAVEN_HOME}/bin
- # mvn -ntp clean install -P${{ matrix.spark }} -Pbackends-velox
-DskipTests
- # - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with
Celeborn 0.4.0
- # run: |
- # export MAVEN_HOME=/usr/lib/maven
- # export PATH=${PATH}:${MAVEN_HOME}/bin
- # cd /opt && mkdir -p celeborn && \
- # wget
https://archive.apache.org/dist/incubator/celeborn/celeborn-0.4.0-incubating/apache-celeborn-0.4.0-incubating-bin.tgz
&& \
- # tar xzf apache-celeborn-0.4.0-incubating-bin.tgz -C /opt/celeborn
--strip-components=1 && cd celeborn && \
- # mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
- # echo -e
"CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g"
> ./conf/celeborn-env.sh && \
- # echo -e "celeborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64" > ./conf/celeborn-defaults.conf \
- # && bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \
- # cd $GITHUB_WORKSPACE/tools/gluten-it && mvn -ntp clean install
-Pspark-3.2,rss,celeborn-0.4 \
- # && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- # --local --preset=velox-with-celeborn --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- # && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- # --local --preset=velox-with-celeborn --benchmark-type=ds
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 && \
- # bash /opt/celeborn/sbin/stop-worker.sh \
- # && bash /opt/celeborn/sbin/stop-master.sh && rm -rf /opt/celeborn
- # - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 with
Celeborn 0.3.2
- # run: |
- # export MAVEN_HOME=/usr/lib/maven
- # export PATH=${PATH}:${MAVEN_HOME}/bin
- # cd /opt && mkdir -p celeborn && \
- # wget
https://archive.apache.org/dist/incubator/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz
&& \
- # tar xzf apache-celeborn-0.3.2-incubating-bin.tgz -C /opt/celeborn
--strip-components=1 && cd celeborn && \
- # mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
- # echo -e
"CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g"
> ./conf/celeborn-env.sh && \
- # echo -e "celeborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64" > ./conf/celeborn-defaults.conf \
- # && bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \
- # cd $GITHUB_WORKSPACE/tools/gluten-it && mvn -ntp clean install
-Pspark-3.2,rss \
- # && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- # --local --preset=velox-with-celeborn --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- # && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- # --local --preset=velox-with-celeborn --benchmark-type=ds
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 && \
- # bash /opt/celeborn/sbin/stop-worker.sh \
- # && bash /opt/celeborn/sbin/stop-master.sh
-
run-spark-test-spark32:
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
@@ -359,7 +291,7 @@ jobs:
- uses: actions/checkout@v2
- name: Build Gluten velox third party
run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y && \
+ yum install sudo patch java-1.8.0-openjdk-devel wget numactl-devel
-y && \
cd ep/build-velox/src && \
./get_velox.sh && \
source /opt/rh/gcc-toolset-9/enable && \
@@ -369,6 +301,10 @@ jobs:
cd $GITHUB_WORKSPACE/cpp && \
source /opt/rh/gcc-toolset-9/enable && \
./compile.sh --build_velox_backend=ON --build_protobuf=ON
--build_tests=ON --build_examples=ON --build_benchmarks=ON
+ - name: Gluten CPP Test
+ run: |
+ cd $GITHUB_WORKSPACE/cpp/build && \
+ ctest -V
- name: Setup java and maven
run: |
wget
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
@@ -401,7 +337,11 @@ jobs:
with:
name: golden-files-spark32
path: /tmp/tpch-approved-plan/**
-
+ - name: Gluten CPP Benchmark Test
+ run: |
+ # This test depends on example.json generated by the above mvn test.
+ cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \
+ ./generic_benchmark --run-example --with-shuffle --threads 1
--iterations 1
run-spark-test-spark32-slow:
runs-on: ubuntu-20.04
@@ -526,7 +466,6 @@ jobs:
export PATH=${PATH}:${MAVEN_HOME}/bin
mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
-
run-spark-test-spark34:
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
