This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bd26d8389 [GLUTEN-6957][VL] Fix missing mvn when CI cache is hit
(#6966)
bd26d8389 is described below
commit bd26d83890466f84c3bf89c218a275cfbe5b73ba
Author: PHILO-HE <[email protected]>
AuthorDate: Thu Aug 22 13:48:36 2024 +0800
[GLUTEN-6957][VL] Fix missing mvn when CI cache is hit (#6966)
---
.github/workflows/util/install_spark_resources.sh | 3 +
.github/workflows/util/setup_helper.sh | 32 +++++++++++
.github/workflows/velox_backend.yml | 67 ++++++-----------------
.github/workflows/velox_backend_cache.yml | 4 ++
dev/ci-velox-buildshared-centos-8.sh | 10 ----
5 files changed, 57 insertions(+), 59 deletions(-)
diff --git a/.github/workflows/util/install_spark_resources.sh
b/.github/workflows/util/install_spark_resources.sh
index 242952d60..e1645b170 100755
--- a/.github/workflows/util/install_spark_resources.sh
+++ b/.github/workflows/util/install_spark_resources.sh
@@ -16,6 +16,9 @@
# Download Spark resources, required by some Spark UTs. The resource path
should be set
# for spark.test.home in mvn test.
+
+set -e
+
INSTALL_DIR=$GITHUB_WORKSPACE
case "$1" in
3.2)
diff --git a/.github/workflows/util/setup_helper.sh
b/.github/workflows/util/setup_helper.sh
new file mode 100644
index 000000000..8b41d8326
--- /dev/null
+++ b/.github/workflows/util/setup_helper.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+function install_maven {
+ (
+ cd /opt/
+ wget -nv
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+ tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8
/usr/lib/maven
+ )
+ echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+}
+
+for cmd in "$@"
+do
+ echo "Running: $cmd"
+ "$cmd"
+done
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index 34ceb6d46..fc375c666 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -44,6 +44,7 @@ env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
MVN_CMD: 'mvn -ntp'
WGET_CMD: 'wget -nv'
+ SETUP: 'bash .github/workflows/util/setup_helper.sh'
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
@@ -206,12 +207,9 @@ jobs:
else
yum update -y && yum install -y java-1.8.0-openjdk-devel wget
fi
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
+ $SETUP install_maven
- name: Set environment variables
run: |
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
if [ "${{ matrix.java }}" = "java-17" ]; then
echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV
elif [ "${{ matrix.java }}" = "java-11" ]; then
@@ -436,13 +434,9 @@ jobs:
- name: Setup java and maven
run: |
yum update -y && yum install -y java-1.8.0-openjdk-devel wget git
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
+ $SETUP install_maven
- name: Build for Uniffle 0.9.0
run: |
- export MAVEN_HOME=/usr/lib/maven && \
- export PATH=${PATH}:${MAVEN_HOME}/bin && \
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
cd /opt && \
git clone -b v0.9.0 https://github.com/apache/incubator-uniffle.git
&& \
@@ -461,14 +455,10 @@ jobs:
bash ./bin/start-coordinator.sh && bash ./bin/start-shuffle-server.sh
- name: Build for Spark ${{ matrix.spark }}
run: |
- export MAVEN_HOME=/usr/lib/maven && \
- export PATH=${PATH}:${MAVEN_HOME}/bin && \
cd $GITHUB_WORKSPACE/ && \
$MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox
-Puniffle -DskipTests
- name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with
uniffle-0.9.0
run: |
- export MAVEN_HOME=/usr/lib/maven && \
- export PATH=${PATH}:${MAVEN_HOME}/bin && \
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
cd $GITHUB_WORKSPACE/tools/gluten-it && \
$MVN_CMD clean install -Pspark-3.2 -Puniffle && \
@@ -554,10 +544,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -614,10 +601,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -661,10 +645,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -722,10 +703,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -770,10 +748,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -831,10 +806,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -879,10 +851,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -939,10 +908,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -993,10 +959,7 @@ jobs:
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
- ${WGET_CMD}
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
- tar -xvf apache-maven-3.8.8-bin.tar.gz
- mv apache-maven-3.8.8 /usr/lib/maven
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ $SETUP install_maven
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -1034,6 +997,12 @@ jobs:
./cpp/build/velox/benchmarks/
/root/.m2/repository/org/apache/arrow/
key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
+ - name: Setup java and maven
+ run: |
+ sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
+ sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
+ yum install sudo patch java-1.8.0-openjdk-devel wget -y
+ $SETUP install_maven
- name: Build Gluten native libraries
if: steps.cache.outputs.cache-hit != 'true'
run: |
diff --git a/.github/workflows/velox_backend_cache.yml
b/.github/workflows/velox_backend_cache.yml
index 0cb922f65..44ee969cb 100644
--- a/.github/workflows/velox_backend_cache.yml
+++ b/.github/workflows/velox_backend_cache.yml
@@ -81,6 +81,10 @@ jobs:
if: steps.check-cache.outputs.cache-hit != 'true'
run: |
df -a
+ sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
+ sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
+ yum install sudo patch java-1.8.0-openjdk-devel wget -y
+ bash .github/workflows/util/setup_helper.sh install_maven
bash dev/ci-velox-buildshared-centos-8.sh
- name: Cache
if: steps.check-cache.outputs.cache-hit != 'true'
diff --git a/dev/ci-velox-buildshared-centos-8.sh
b/dev/ci-velox-buildshared-centos-8.sh
index f337185de..b6b0cda02 100755
--- a/dev/ci-velox-buildshared-centos-8.sh
+++ b/dev/ci-velox-buildshared-centos-8.sh
@@ -2,16 +2,6 @@
set -e
-sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
-sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
-
-yum install sudo patch java-1.8.0-openjdk-devel wget -y
-# Required by building arrow java.
-wget
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
-tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven
-export PATH="${PATH}:/usr/lib/maven/bin"
-echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
-
source /opt/rh/gcc-toolset-9/enable
./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_tests=ON \
--build_examples=ON --build_benchmarks=ON --build_protobuf=ON
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
