This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 1845f30c4 [VL] Add a docker build job and use pre-installed arrow libs
(#6826)
1845f30c4 is described below
commit 1845f30c4c36dc26178c87752ebad2a8897e2b4c
Author: PHILO-HE <[email protected]>
AuthorDate: Thu Aug 15 21:25:08 2024 +0800
[VL] Add a docker build job and use pre-installed arrow libs (#6826)
---
.github/workflows/build_bundle_package.yml | 1 -
.github/workflows/docker_image.yml | 48 ++++++++++++++++++++++++++++++
.github/workflows/velox_docker.yml | 3 +-
.github/workflows/velox_docker_cache.yml | 4 +--
dev/ci-velox-buildstatic-centos-7.sh | 7 ++---
dev/vcpkg/docker/Dockerfile.gha | 15 ++++++----
dev/vcpkg/env.sh | 1 +
7 files changed, 63 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/build_bundle_package.yml
b/.github/workflows/build_bundle_package.yml
index 950f07ce2..cf70eeb1a 100644
--- a/.github/workflows/build_bundle_package.yml
+++ b/.github/workflows/build_bundle_package.yml
@@ -52,7 +52,6 @@ jobs:
source /opt/rh/devtoolset-9/enable && \
source /opt/gluten//dev/vcpkg/env.sh && \
cd $GITHUB_WORKSPACE/ && \
- sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt &&
\
export NUM_THREADS=4
./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF
--enable_s3=OFF \
--enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF
diff --git a/.github/workflows/docker_image.yml
b/.github/workflows/docker_image.yml
new file mode 100644
index 000000000..7eecfdecc
--- /dev/null
+++ b/.github/workflows/docker_image.yml
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build and Push Docker Image
+
+on:
+ pull_request:
+ paths:
+ - '.github/workflows/docker_image.yml'
+ schedule:
+ - cron: '0 20 * * 0'
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USER }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Build and push Docker image
+ uses: docker/build-push-action@v2
+ with:
+ context: .
+ file: dev/vcpkg/docker/Dockerfile.gha
+ push: true
+ tags: apache/gluten:vcpkg-centos-7
diff --git a/.github/workflows/velox_docker.yml
b/.github/workflows/velox_docker.yml
index 717cb5d2d..c310d5298 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -51,7 +51,7 @@ concurrency:
jobs:
build-native-lib-centos-7:
runs-on: ubuntu-20.04
- container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with
dependencies installed
+ container: apache/gluten:vcpkg-centos-7
steps:
- uses: actions/checkout@v2
- name: Generate cache key
@@ -63,7 +63,6 @@ jobs:
with:
path: |
./cpp/build/releases/
- /root/.m2/repository/org/apache/arrow/
key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
- name: Build Gluten native libraries
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
diff --git a/.github/workflows/velox_docker_cache.yml
b/.github/workflows/velox_docker_cache.yml
index 3fc0fc50f..1aca4d6bd 100644
--- a/.github/workflows/velox_docker_cache.yml
+++ b/.github/workflows/velox_docker_cache.yml
@@ -30,7 +30,7 @@ concurrency:
jobs:
cache-native-lib-centos-7:
runs-on: ubuntu-20.04
- container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with
dependencies installed
+ container: apache/gluten:vcpkg-centos-7
steps:
- uses: actions/checkout@v2
- name: Generate cache key
@@ -43,7 +43,6 @@ jobs:
lookup-only: true
path: |
./cpp/build/releases/
- /root/.m2/repository/org/apache/arrow/
key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
- name: Build Gluten native libraries
if: steps.check-cache.outputs.cache-hit != 'true'
@@ -57,7 +56,6 @@ jobs:
with:
path: |
./cpp/build/releases/
- /root/.m2/repository/org/apache/arrow/
key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
cache-native-lib-centos-8:
diff --git a/dev/ci-velox-buildstatic-centos-7.sh
b/dev/ci-velox-buildstatic-centos-7.sh
index 50f8ee067..d83d443fc 100755
--- a/dev/ci-velox-buildstatic-centos-7.sh
+++ b/dev/ci-velox-buildstatic-centos-7.sh
@@ -2,12 +2,9 @@
set -e
-yum install sudo patch java-1.8.0-openjdk-devel -y
-cd $GITHUB_WORKSPACE/ep/build-velox/src
-./get_velox.sh
source /opt/rh/devtoolset-9/enable
cd $GITHUB_WORKSPACE/
source ./dev/vcpkg/env.sh
-sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt
export NUM_THREADS=4
-./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF
--enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
+./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF
--build_arrow=OFF --enable_s3=ON \
+ --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
diff --git a/dev/vcpkg/docker/Dockerfile.gha b/dev/vcpkg/docker/Dockerfile.gha
index c54725f04..d37c449cb 100644
--- a/dev/vcpkg/docker/Dockerfile.gha
+++ b/dev/vcpkg/docker/Dockerfile.gha
@@ -11,15 +11,20 @@ RUN sed -i \
-e 's/mirror\.centos\.org/vault.centos.org/' \
/etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
-RUN yum install -y git patch wget sudo
+RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel
-# build
RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
RUN echo "check_certificate = off" >> ~/.wgetrc
-# deps
+
RUN cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh
-# vcpkg env
+# An actual path used for vcpkg cache.
+RUN mkdir -p /var/cache/vcpkg
+
+# Set vcpkg cache path.
ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite
-RUN source /opt/rh/devtoolset-9/enable && cd /opt/gluten && source
dev/vcpkg/env.sh
+
+# Build arrow, then install the native libs to system paths and jar package to
.m2/ directory.
+RUN cd /opt/gluten && source /opt/rh/devtoolset-9/enable && source
./dev/vcpkg/env.sh && \
+ bash ./dev/builddeps-veloxbe.sh build_arrow && rm -rf /opt/gluten
diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh
index cf3126f0a..8b247a907 100755
--- a/dev/vcpkg/env.sh
+++ b/dev/vcpkg/env.sh
@@ -1,4 +1,5 @@
#! /bin/bash
+set -e
if [ -z "${BASH_SOURCE[0]}" ] || [ "$0" == "${BASH_SOURCE[0]}" ]; then
echo "env.sh should only be sourced in bash" >&2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
