This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2f2e66266 [VL] Minor: Remove deprecated GHA jobs
2f2e66266 is described below
commit 2f2e662667a74f03e17b68878b2b7db8fa76d1aa
Author: PHILO-HE <[email protected]>
AuthorDate: Fri Jul 26 16:51:22 2024 +0800
[VL] Minor: Remove deprecated GHA jobs
---
.github/workflows/velox_be.yml.deprecated | 700 ------------------------
.github/workflows/velox_tpch_bench.yml.disabled | 57 --
.github/workflows/velox_tpch_merge.yml.disabled | 60 --
.github/workflows/velox_velox_ut.yml.disabled | 67 ---
4 files changed, 884 deletions(-)
diff --git a/.github/workflows/velox_be.yml.deprecated
b/.github/workflows/velox_be.yml.deprecated
deleted file mode 100644
index 6ff5ec743..000000000
--- a/.github/workflows/velox_be.yml.deprecated
+++ /dev/null
@@ -1,700 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Velox backend
-
-on:
- pull_request:
- paths:
- - '.github/**'
- - 'pom.xml'
- - 'backends-velox/**'
- - 'gluten-celeborn/**'
- - 'gluten-core/**'
- - 'gluten-data/**'
- - 'gluten-delta/**'
- - 'gluten-iceberg/**'
- - 'gluten-ut/**'
- - 'shims/**'
- - 'tools/gluten-it/**'
- - 'tools/gluten-te/**'
- - 'ep/build-velox/**'
- - 'cpp/*'
- - 'cpp/CMake/**'
- - 'cpp/velox/**'
- - 'cpp/core/**'
- - 'dev/**'
-
-env:
- HTTP_PROXY_HOST: proxy-shz.intel.com
- HTTP_PROXY_PORT: 911
- PATH_TO_GLUTEN_TE: ./tools/gluten-te
- DOCKER_PULL_REGISTRY: 10.1.0.25:5000
- MAVEN_OPTS: -Dmaven.wagon.http.retryHandler.count=3
-
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-jobs:
- ubuntu2004-test-spark32:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 20.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF
--build_test_utils=ON'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON --build_tests=ON
--build_examples=ON --build_benchmarks=ON'
- - name: Run CPP unit test
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/cpp/build && \
- ctest -V'
- - name: Run HBM CPP unit test
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/cpp/build && \
- cmake -DBUILD_TESTS=ON -DENABLE_HBM=ON .. && \
- cmake --build . --target hbw_allocator_test -- -j && \
- ctest -V -R TestHbw'
- - name: Build and run unit test for Spark 3.2.2 (other tests)
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss
-DargLine="-Dspark.test.home=/opt/spark322"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
&& \
- mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=org.apache.gluten.tags.UDFTest'
- # Cpp micro benchmarks will use generated files from unit test in
backends-velox module.
- - name: Run micro benchmarks
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/cpp/build/velox/benchmarks && \
- ./generic_benchmark --run-example --with-shuffle --threads 1
--iterations 1'
- - name: Copy golden files from container to host
- if: failure()
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/cp.sh
/tmp/tpch-approved-plan/ /tmp/$GITHUB_RUN_ID/spark32/tpch-approved-plan
- - name: Upload golden files
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: golden-files-spark32
- path: |
- /tmp/${{ github.run_id }}/spark32/tpch-approved-plan/**
- - name: Clean temp golden files
- run: |
- rm -rf /tmp/$GITHUB_RUN_ID/spark32
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- ubuntu2004-test-spark32-slow:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 20.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON'
- - name: Build and run unit test for Spark 3.2.2 (slow tests)
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss
-Piceberg -Pdelta -DargLine="-Dspark.test.home=/opt/spark322"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest'
- - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.2 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- ubuntu2004-test-spark33-slow:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 20.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON'
- - name: Build and Run unit test for Spark 3.3.1 (slow tests)
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten && \
- mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest'
- - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.3 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1'
- - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 Q38 flush
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 --queries=q38 \
- --disable-bhj \
-
--extra-conf=spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.1
\
-
--extra-conf=spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.2
\
-
--extra-conf=spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100
\
-
--extra-conf=spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- ubuntu2004-test-spark33:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 20.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON --build_examples=ON'
- - name: Build and Run unit test for Spark 3.3.1 (other tests)
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten && \
- mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
&& \
- mvn test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=org.apache.gluten.tags.UDFTest'
- - name: Copy golden files from container to host
- if: failure()
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/cp.sh
/tmp/tpch-approved-plan/ /tmp/$GITHUB_RUN_ID/spark33/tpch-approved-plan
- - name: Upload golden files
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: golden-files-spark33
- path: |
- /tmp/${{ github.run_id }}/spark33/tpch-approved-plan/**
- - name: Clean temp golden files
- run: |
- rm -rf /tmp/$GITHUB_RUN_ID/spark33
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- ubuntu2004-test-spark34-slow:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 20.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON '
- - name: Build and Run unit test for Spark 3.4.2 (slow tests)
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten && \
- mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark342"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest'
- - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.4
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.4 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- ubuntu2004-test-spark34:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 20.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON --build_examples=ON'
- - name: Build and Run unit test for Spark 3.4.2 (other tests)
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten && \
- mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark342"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
&& \
- mvn test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=org.apache.gluten.tags.UDFTest'
- - name: Copy golden files from container to host
- if: failure()
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/cp.sh
/tmp/tpch-approved-plan/ /tmp/$GITHUB_RUN_ID/spark34/tpch-approved-plan
- - name: Upload golden files
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: golden-files-spark34
- path: |
- /tmp/${{ github.run_id }}/spark34/tpch-approved-plan/**
- - name: Clean temp golden files
- run: |
- rm -rf /tmp/$GITHUB_RUN_ID/spark34
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- ubuntu2004-test-spark35:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 20.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON '
- - name: Build for Spark 3.5
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.5 -Pbackends-velox -Prss -DskipTests'
- - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.5
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.5 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- ubuntu2204-test-spark33-spark34:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 22.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON
--enable_abfs=ON && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF
--enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON --enable_hdfs=ON
--enable_s3=ON --enable_gcs=ON --enable_abfs=ON'
- - name: Build for Spark 3.3.1
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg
-Pdelta -DskipTests'
- - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.3
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.3 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=30g -s=10.0 --threads=32 --iterations=1'
- - name: Build for Spark 3.4.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg
-Pdelta -DskipTests'
- - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.4
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.4 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=30g -s=10.0 --threads=32 --iterations=1'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- ubuntu2204-test:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: ubuntu
- OS_IMAGE_TAG: 22.04
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON
--enable_abfs=ON && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF
--enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON --enable_hdfs=ON
--enable_s3=ON --enable_gcs=ON --enable_abfs=ON'
- - name: Build for Spark 3.2.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg
-Pdelta -DskipTests'
- - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.2 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=40g -s=10.0 --threads=32 --iterations=1'
- - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 with
Celeborn 0.4.0
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh \
- 'cd /opt && mkdir -p celeborn && \
- tar xzf apache-celeborn-0.4.0-incubating-bin.tgz -C /opt/celeborn
--strip-components=1 && cd celeborn && \
- mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
- echo -e
"CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g"
> ./conf/celeborn-env.sh && \
- echo -e "celeborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64" > ./conf/celeborn-defaults.conf \
- && bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \
- cd /opt/gluten/tools/gluten-it && mvn clean install
-Pspark-3.2,rss,celeborn-0.4 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox-with-celeborn --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox-with-celeborn --benchmark-type=ds
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 && \
- bash /opt/celeborn/sbin/stop-worker.sh \
- && bash /opt/celeborn/sbin/stop-master.sh && rm -rf /opt/celeborn'
- - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 with
Celeborn 0.3.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh \
- 'cd /opt && mkdir -p celeborn && \
- tar xzf apache-celeborn-0.3.2-incubating-bin.tgz -C /opt/celeborn
--strip-components=1 && cd celeborn && \
- mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
- echo -e
"CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g"
> ./conf/celeborn-env.sh && \
- echo -e "celeborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64" > ./conf/celeborn-defaults.conf \
- && bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \
- cd /opt/gluten/tools/gluten-it && mvn clean install -Pspark-3.2,rss \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox-with-celeborn --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox-with-celeborn --benchmark-type=ds
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 && \
- bash /opt/celeborn/sbin/stop-worker.sh \
- && bash /opt/celeborn/sbin/stop-master.sh'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- centos8-test:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: centos
- OS_IMAGE_TAG: 8
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- source /env.sh && \
- sudo yum -y install patch && \
- cd /opt/gluten/ep/build-velox/src && \
- ./get_velox.sh --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON
--enable_abfs=ON && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF
--enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- source /env.sh && \
- cd /opt/gluten/cpp && \
- ./compile.sh --build_velox_backend=ON --enable_hdfs=ON
--enable_s3=ON --enable_gcs=ON --enable_abfs=ON'
- - name: Build for Spark 3.2.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg
-Pdelta -DskipTests'
- - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.2 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=50g -s=30.0 --threads=32 --iterations=1'
- - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 random kill
tasks
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.2 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 --data-gen-strategy=skip
--random-kill-tasks \
- && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh queries \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=50g -s=30.0 --threads=32 --iterations=1
--data-gen-strategy=skip --random-kill-tasks'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
- centos7-test:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: centos
- OS_IMAGE_TAG: 7
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Gluten velox third party
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- yum -y install epel-release centos-release-scl patch sudo && \
- cd /opt/gluten/ep/build-velox/src && \
- source /opt/rh/devtoolset-9/enable && \
- ./get_velox.sh --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON
--enable_abfs=ON && \
- ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF
--enable_s3=ON --enable_gcs=ON --enable_abfs=ON --enable_hdfs=ON'
- - name: Build Gluten CPP library
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten/cpp && \
- source /opt/rh/devtoolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --enable_hdfs=ON
--enable_s3=ON --enable_gcs=ON --enable_abfs=ON'
- - name: Build for Spark 3.2.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg
-Pdelta -DskipTests'
- - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.2 \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=50g -s=30.0 --threads=32 --iterations=1'
- - name: TPC-DS SF30.0 Parquet local spark3.2 Q67/Q95 low memory, memory
isolation off
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.2 \
- && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
- --data-gen-strategy=skip -m=OffHeapExecutionMemory \
- -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
- -d=OFFHEAP_SIZE:5g,spark.memory.offHeap.size=5g \
- -d=OFFHEAP_SIZE:3g,spark.memory.offHeap.size=3g \
-
-d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \
-
-d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5'
- - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q67/Q95 low
memory, memory isolation on
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.2 \
- && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
- --data-gen-strategy=skip -m=OffHeapExecutionMemory \
-
-d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1
\
- -d=OFFHEAP_SIZE:5g,spark.memory.offHeap.size=5g \
- -d=OFFHEAP_SIZE:3g,spark.memory.offHeap.size=3g \
-
-d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \
-
-d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5' || true
- - name: TPC-DS SF30.0 Parquet local spark3.2 Q23A/Q23B low memory
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1
\
- --data-gen-strategy=skip -m=OffHeapExecutionMemory \
- -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
-
-d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1
\
- -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
-
-d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0
\
-
-d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0
\
-
-d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0'
- - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q97 low
memory # The case currently causes crash with "free: invalid size".
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd
/opt/gluten/tools/gluten-it && \
- GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--queries=q97 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
- --data-gen-strategy=skip -m=OffHeapExecutionMemory \
- -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
-
-d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1
\
- -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
- -d=OFFHEAP_SIZE:1g,spark.memory.offHeap.size=1g' || true
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
-
-
- static-build-centos7-test:
- runs-on: velox-self-hosted
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- docker run --rm --init --privileged --ulimit nofile=65536:65536
--ulimit core=-1 --security-opt seccomp=unconfined \
- -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID -e
NUM_THREADS=30 -detach $DOCKER_PULL_REGISTRY/gluten-te/gluten-buildenv-centos:7
\
- bash -c 'cd /opt/gluten && sleep 14400'
- - name: Build Gluten CPP library
- run: |
- docker exec -i static-build-test-$GITHUB_RUN_ID bash -c '
- source /env.sh && \
- sudo yum -y install patch && \
- cd /opt/gluten && \
- sudo -E ./dev/vcpkg/setup-build-depends.sh && \
- source ./dev/vcpkg/env.sh && \
- ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=OFF
--build_benchmarks=OFF --enable_s3=ON \
- --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON --build_type=Debug'
- - name: Build for Spark 3.2.2
- run: |
- docker exec static-build-test-$GITHUB_RUN_ID bash -c '
- cd /opt/gluten && \
- mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg
-Pdelta -DskipTests && \
- cd /opt/gluten/tools/gluten-it && \
- mvn clean install -Pspark-3.2'
- - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (ubuntu 20.04)
- run: |
- docker run --rm --init --privileged --ulimit nofile=65536:65536
--ulimit core=-1 --security-opt seccomp=unconfined \
- -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e
NUM_THREADS=30 ubuntu:20.04 \
- bash -c 'apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get
install openjdk-8-jre -y \
- && cd /opt/gluten/tools/gluten-it \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=20g -s=1.0 --threads=32 --iterations=1'
- - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (ubuntu 22.04)
- run: |
- docker run --rm --init --privileged --ulimit nofile=65536:65536
--ulimit core=-1 --security-opt seccomp=unconfined \
- -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e
NUM_THREADS=30 ubuntu:22.04 \
- bash -c 'apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get
install openjdk-8-jre -y \
- && cd /opt/gluten/tools/gluten-it \
- && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=20g -s=1.0 --threads=32 --iterations=1'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- docker stop static-build-test-$GITHUB_RUN_ID || true
-
- build-script-test:
- runs-on: velox-self-hosted
- env:
- OS_IMAGE_NAME: centos
- OS_IMAGE_TAG: 8
- steps:
- - uses: actions/checkout@v4
- - name: Setup docker container
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh
- - name: Build Script Test
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
- source /env.sh && \
- sudo yum -y install patch && \
- cd /opt/gluten/ && \
- ./dev/package.sh'
- - name: Exit docker container
- if: ${{ always() }}
- run: |
- $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh
diff --git a/.github/workflows/velox_tpch_bench.yml.disabled
b/.github/workflows/velox_tpch_bench.yml.disabled
deleted file mode 100644
index 6be12305c..000000000
--- a/.github/workflows/velox_tpch_bench.yml.disabled
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Velox backend Performance Tests On Comments
-
-on:
- issue_comment:
- types: [created, edited, deleted]
-
-env:
- JENKINS_HOST: 10.0.0.68
- JENKINS_PORT: 8080
- JENKINS_USER: jenkins:11fd1b5a82bfd638bd9b3749c96b324ff2
-
-jobs:
- velox-trigger-benchmark-on-comment:
- # check the comment if it contains the keywords
- if: ${{ github.event.issue.pull_request &&
contains(github.event.comment.body, '/Benchmark Velox') }}
- runs-on: velox-self-hosted
- container: ubuntu:22.04
- steps:
- - uses: actions/checkout@v4
- - run: apt-get update && apt-get install ca-certificates -y &&
update-ca-certificates
- - run: sed -i
's/http\:\/\/archive.ubuntu.com/https\:\/\/mirrors.ustc.edu.cn/g'
/etc/apt/sources.list
- - run: apt-get update
- - run: apt-get install -y curl
- - name: trigger_tpch_bench
- run: |
- branchname=origin/pr/${{ github.event.issue.number }}/head
- curl --noproxy $JENKINS_HOST -L -X POST
http://$JENKINS_HOST:$JENKINS_PORT/job/Gluten_Perf_Github_Action_Check/build
--user $JENKINS_USER --data-urlencode json='{"parameter": [{"name":"sha1",
"value":"'$branchname'"}]}'
- velox-trigger-tpcds-on-comment:
- # check the comment if it contains the keywords
- if: ${{ github.event.issue.pull_request &&
contains(github.event.comment.body, '/Benchmark Velox TPCDS') }}
- runs-on: velox-self-hosted
- container: ubuntu:22.04
- steps:
- - uses: actions/checkout@v4
- - run: apt-get update && apt-get install ca-certificates -y &&
update-ca-certificates
- - run: sed -i
's/http\:\/\/archive.ubuntu.com/https\:\/\/mirrors.ustc.edu.cn/g'
/etc/apt/sources.list
- - run: apt-get update
- - run: apt-get install -y curl
- - name: trigger_tpch_bench
- run: |
- branchname=origin/pr/${{ github.event.issue.number }}/head
- curl --noproxy $JENKINS_HOST -L -X POST
http://$JENKINS_HOST:$JENKINS_PORT/job/Gluten_Perf_DS_Github_Action_Check/build
--user $JENKINS_USER --data-urlencode json='{"parameter": [{"name":"sha1",
"value":"'$branchname'"}]}'
diff --git a/.github/workflows/velox_tpch_merge.yml.disabled
b/.github/workflows/velox_tpch_merge.yml.disabled
deleted file mode 100644
index 5012e8301..000000000
--- a/.github/workflows/velox_tpch_merge.yml.disabled
+++ /dev/null
@@ -1,60 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Velox backend Performance Tests on Merge
-
-on:
- pull_request:
- types: [ closed ]
- branches:
- - main
- paths:
- - '.github/**'
- - 'pom.xml'
- - 'backends-velox/**'
- - 'gluten-celeborn/**'
- - 'gluten-core/**'
- - 'gluten-data/**'
- - 'gluten-ut/**'
- - 'shims/**'
- - 'tools/gluten-it/**'
- - 'tools/gluten-te/**'
- - 'ep/build-velox/**'
- - 'cpp/*'
- - 'cpp/CMake/**'
- - 'cpp/velox/**'
- - 'cpp/core/**'
- - 'dev/**'
-
-env:
- JENKINS_HOST: 10.0.0.68
- JENKINS_PORT: 8080
- JENKINS_USER: jenkins:11fd1b5a82bfd638bd9b3749c96b324ff2
-
-jobs:
- velox-trigger-benchmark:
- if: github.event.pull_request.merged == true
- runs-on: velox-self-hosted
- container: ubuntu:22.04
- steps:
- - uses: actions/checkout@v4
- - run: apt-get update && apt-get install ca-certificates -y &&
update-ca-certificates
- - run: sed -i
's/http\:\/\/archive.ubuntu.com/https\:\/\/mirrors.ustc.edu.cn/g'
/etc/apt/sources.list
- - run: apt-get update
- - run: apt-get install -y curl
- - name: trigger_tpch_bench
- run: |
- branchname=origin/pr/${{github.event.number}}/head
- curl --noproxy $JENKINS_HOST -L -X POST
http://$JENKINS_HOST:$JENKINS_PORT/job/Gluten_Perf_Github_Action_Check/build
--user $JENKINS_USER --data-urlencode json='{"parameter": [{"name":"sha1",
"value":"'$branchname'"}]}'
diff --git a/.github/workflows/velox_velox_ut.yml.disabled
b/.github/workflows/velox_velox_ut.yml.disabled
deleted file mode 100644
index efdf51298..000000000
--- a/.github/workflows/velox_velox_ut.yml.disabled
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Velox backend Velox Unit test
-
-
-on:
- pull_request:
- paths:
- - '.github/workflows/velox_velox_ut.yml'
- - 'dev/**'
- - 'ep/**' #get_velox change
-
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-jobs:
- run-velox-unit-test:
- runs-on: ubuntu-20.04
- env:
- CCACHE_DIR: "${{ github.workspace }}/.ccache"
- container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx
- steps:
- - uses: actions/checkout@v2
- - name: Get Ccache
- uses: actions/cache/restore@v3
- with:
- path: '${{ env.CCACHE_DIR }}'
- key: ccache-ubuntu-release-default
- - name: Ensure Cache Dirs Exists
- working-directory: ${{ github.workspace }}
- run: |
- mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- rm -rf /opt/miniconda-for-velox/
- cd ep/build-velox/src && \
- ./get_velox.sh
- cd ../build/velox_ep/
- git reset --hard
- make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON
-DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON"
-
- - name: CCache after
- run: |
- ccache -vs
-
- - uses: actions/cache/save@v3
- with:
- path: '${{ env.CCACHE_DIR }}'
- key: ccache-ubuntu-release-default
-
- - name: Run Tests
- run: |
- cd ${GITHUB_WORKSPACE}/ep/build-velox/build/velox_ep/_build/release
&& ctest -E "velox_cache_test|velox_exec_test" -j 4 --output-on-failure
--no-tests=error
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
