This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 518612cab [VL] Fix broken GHA CI cache, add cache for Centos 8 build
(#6497)
518612cab is described below
commit 518612cab6d6894877086312aaac10b7503d4a0c
Author: Hongze Zhang <[email protected]>
AuthorDate: Fri Jul 19 11:39:19 2024 +0800
[VL] Fix broken GHA CI cache, add cache for Centos 8 build (#6497)
---
.github/workflows/velox_docker.yml | 35 +++++----------
.github/workflows/velox_docker_cache.yml | 51 +++++++++++++++++++---
dev/ci-velox-buildshared-centos-8.sh | 18 ++++++++
...dstatic.sh => ci-velox-buildstatic-centos-7.sh} | 4 ++
.../gluten/planner/cost/GlutenCostModel.scala | 3 +-
5 files changed, 78 insertions(+), 33 deletions(-)
diff --git a/.github/workflows/velox_docker.yml
b/.github/workflows/velox_docker.yml
index 592552a19..ea28139a3 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -63,13 +63,13 @@ jobs:
with:
path: |
./cpp/build/releases/
- ~/.m2/repository/org/apache/arrow/
- key: cache-velox-build-${{ hashFiles('./cache-key') }}
- - name: Build Gluten Velox third party
+ /root/.m2/repository/org/apache/arrow/
+ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+ - name: Build Gluten native libraries
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
run: |
df -a
- source dev/ci-velox-buildstatic.sh
+ bash dev/ci-velox-buildstatic-centos-7.sh
- name: Upload Artifact Native
uses: actions/upload-artifact@v2
with:
@@ -597,30 +597,15 @@ jobs:
with:
path: |
./cpp/build/releases/
+ ./cpp/build/velox/udf/examples/
+ ./cpp/build/velox/benchmarks/
/root/.m2/repository/org/apache/arrow/
key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+ - name: Build Gluten native libraries
+ if: steps.check-cache.outputs.cache-hit != 'true'
run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- # Required by building arrow java.
- wget
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8
/usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
- - name: Build Gluten Velox third party
- if: ${{ steps.cache.outputs.cache-hit != 'true' }}
- run: |
- source /opt/rh/gcc-toolset-9/enable
- ./dev/builddeps-veloxbe.sh --run_setup_script=OFF
--enable_ep_cache=OFF --build_tests=ON \
- --build_examples=ON --build_benchmarks=ON --build_protobuf=ON
- - name: Gluten CPP Test
- run: |
- cd ./cpp/build && \
- ctest -V
+ df -a
+ bash dev/ci-velox-buildshared-centos-8.sh
- uses: actions/upload-artifact@v2
with:
name: velox-native-lib-centos-8-${{github.sha}}
diff --git a/.github/workflows/velox_docker_cache.yml
b/.github/workflows/velox_docker_cache.yml
index 6e8b7c1d0..f71583ea3 100644
--- a/.github/workflows/velox_docker_cache.yml
+++ b/.github/workflows/velox_docker_cache.yml
@@ -28,7 +28,7 @@ concurrency:
cancel-in-progress: false
jobs:
- cache-native-lib:
+ cache-native-lib-centos-7:
runs-on: ubuntu-20.04
container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with
dependencies installed
steps:
@@ -44,11 +44,12 @@ jobs:
path: |
./cpp/build/releases/
/root/.m2/repository/org/apache/arrow/
- key: cache-velox-build-${{ hashFiles('./cache-key') }}
- - name: Build Gluten Velox third party
+ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+ - name: Build Gluten native libraries
if: steps.check-cache.outputs.cache-hit != 'true'
run: |
- source dev/ci-velox-buildstatic.sh
+ df -a
+ bash dev/ci-velox-buildstatic-centos-7.sh
- name: Cache
if: steps.check-cache.outputs.cache-hit != 'true'
id: cache
@@ -57,7 +58,43 @@ jobs:
path: |
./cpp/build/releases/
/root/.m2/repository/org/apache/arrow/
- key: cache-velox-build-${{ hashFiles('./cache-key') }}
+ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+
+ cache-native-lib-centos-8:
+ runs-on: ubuntu-20.04
+ container: ghcr.io/facebookincubator/velox-dev:centos8
+ steps:
+ - uses: actions/checkout@v2
+ - name: Generate cache key
+ run: |
+ echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*',
'./github/workflows/*') }} > cache-key
+ - name: Check existing caches
+ id: check-cache
+ uses: actions/cache/restore@v3
+ with:
+ lookup-only: true
+ path: |
+ ./cpp/build/releases/
+ ./cpp/build/velox/udf/examples/
+ ./cpp/build/velox/benchmarks/
+ /root/.m2/repository/org/apache/arrow/
+ key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
+ - name: Build Gluten native libraries
+ if: steps.check-cache.outputs.cache-hit != 'true'
+ run: |
+ df -a
+ bash dev/ci-velox-buildstatic-centos-8.sh
+ - name: Cache
+ if: steps.check-cache.outputs.cache-hit != 'true'
+ id: cache
+ uses: actions/cache/save@v3
+ with:
+ path: |
+ ./cpp/build/releases/
+ ./cpp/build/velox/udf/examples/
+ ./cpp/build/velox/benchmarks/
+ /root/.m2/repository/org/apache/arrow/
+ key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
# ccache-native-lib-ubuntu-velox-ut:
# runs-on: ubuntu-20.04
@@ -75,7 +112,7 @@ jobs:
# working-directory: ${{ github.workspace }}
# run: |
# mkdir -p '${{ env.CCACHE_DIR }}'
- # - name: Build Gluten velox third party
+ # - name: Build Gluten native libraries
# run: |
# rm -rf /opt/miniconda-for-velox/
# cd ep/build-velox/src && \
@@ -113,7 +150,7 @@ jobs:
# working-directory: ${{ github.workspace }}
# run: |
# mkdir -p '${{ env.CCACHE_DIR }}'
-# - name: Build Gluten velox third party
+# - name: Build Gluten native libraries
# run: |
# rm -rf /opt/miniconda-for-velox/
# cd ep/build-velox/src && \
diff --git a/dev/ci-velox-buildshared-centos-8.sh
b/dev/ci-velox-buildshared-centos-8.sh
new file mode 100755
index 000000000..362900bd0
--- /dev/null
+++ b/dev/ci-velox-buildshared-centos-8.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
+sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
+
+yum install sudo patch java-1.8.0-openjdk-devel wget -y
+# Required by building arrow java.
+wget
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven
+export PATH="${PATH}:/usr/lib/maven/bin"
+
+source /opt/rh/gcc-toolset-9/enable
+./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_tests=ON \
+ --build_examples=ON --build_benchmarks=ON --build_protobuf=ON
+
+cd ./cpp/build && ctest -V
diff --git a/dev/ci-velox-buildstatic.sh b/dev/ci-velox-buildstatic-centos-7.sh
similarity index 95%
rename from dev/ci-velox-buildstatic.sh
rename to dev/ci-velox-buildstatic-centos-7.sh
index 227bad360..50f8ee067 100755
--- a/dev/ci-velox-buildstatic.sh
+++ b/dev/ci-velox-buildstatic-centos-7.sh
@@ -1,3 +1,7 @@
+#!/bin/bash
+
+set -e
+
yum install sudo patch java-1.8.0-openjdk-devel -y
cd $GITHUB_WORKSPACE/ep/build-velox/src
./get_velox.sh
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala
b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala
index 513a91e43..c028a9559 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala
@@ -71,7 +71,8 @@ object GlutenCostModel extends Logging {
}
// A very rough estimation as of now. The cost model basically considers
any
- // fallen back ops has extreme high cost so offloads computations as much
as possible.
+ // fallen back ops as having extreme high cost so offloads computations as
+ // much as possible.
private def selfLongCostOf(node: SparkPlan): Long = {
node match {
case _: RemoveFilter.NoopFilter =>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
