This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch branch-1.2
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/branch-1.2 by this push:
new 54d66ae256 [VL][Branch-1.2] Port #8034 & #8027 for fixing march flag
and #8042 for fixing GHA failure on centos-7 (#8075)
54d66ae256 is described below
commit 54d66ae25679a81b8ca2797aed9aa092a2dc4c9c
Author: PHILO-HE <[email protected]>
AuthorDate: Thu Nov 28 20:32:14 2024 +0800
[VL][Branch-1.2] Port #8034 & #8027 for fixing march flag and #8042 for
fixing GHA failure on centos-7 (#8075)
---
.github/workflows/velox_backend.yml | 69 +++++++++++++++++++++----------------
cpp/CMakeLists.txt | 21 ++++++++++-
cpp/core/CMakeLists.txt | 10 +-----
cpp/velox/CMakeLists.txt | 18 ----------
ep/build-velox/src/get_velox.sh | 1 -
5 files changed, 61 insertions(+), 58 deletions(-)
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index 95445aec3d..8bd9ac2fe0 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -47,6 +47,7 @@ env:
MVN_CMD: 'mvn -ntp'
WGET_CMD: 'wget -nv'
SETUP: 'bash .github/workflows/util/setup_helper.sh'
+ CCACHE_DIR: "${{ github.workspace }}/.ccache"
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
@@ -55,26 +56,36 @@ concurrency:
jobs:
build-native-lib-centos-7:
runs-on: ubuntu-20.04
- container: apache/gluten:vcpkg-centos-7
steps:
- - uses: actions/checkout@v2
- - name: Generate cache key
- run: |
- echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*',
'./.github/workflows/*') }} > cache-key
- - name: Cache
- id: cache
+ - uses: actions/checkout@v4
+ - name: Get Ccache
uses: actions/cache/restore@v3
with:
- path: |
- ./cpp/build/releases/
- key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-centos7-release-default-${{github.sha}}
+ restore-keys: |
+ ccache-centos7-release-default
- name: Build Gluten native libraries
- if: ${{ steps.cache.outputs.cache-hit != 'true' }}
run: |
- df -a
- yum install -y devtoolset-9
- cd $GITHUB_WORKSPACE/
- bash dev/ci-velox-buildstatic-centos-7.sh
+ docker pull apache/gluten:vcpkg-centos-7
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7 bash -c "
+ df -a
+ yum install -y devtoolset-9
+ cd /work
+ export CCACHE_DIR=/work/.ccache
+ mkdir -p /work/.ccache
+ bash dev/ci-velox-buildstatic-centos-7.sh
+ ccache -s
+ mkdir -p /work/.m2/repository/org/apache/arrow/
+ cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
+ "
+
+ - name: "Save ccache"
+ uses: actions/cache/save@v3
+ id: ccache
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-centos7-release-default-${{github.sha}}
- uses: actions/upload-artifact@v3
with:
name: velox-native-lib-centos-7-${{github.sha}}
@@ -82,7 +93,7 @@ jobs:
- uses: actions/upload-artifact@v3
with:
name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
+ path: .m2/repository/org/apache/arrow/
run-tpc-test-ubuntu:
needs: build-native-lib-centos-7
@@ -160,7 +171,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [ "centos:7", "centos:8" ]
+ os: [ "centos:8" ]
spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ]
java: [ "java-8", "java-11", "java-17" ]
# Spark supports JDK17 since 3.3 and later, see
https://issues.apache.org/jira/browse/SPARK-33772
@@ -1036,19 +1047,13 @@ jobs:
container: ghcr.io/facebookincubator/velox-dev:centos8
steps:
- uses: actions/checkout@v2
- - name: Generate cache key
- run: |
- echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*',
'./.github/workflows/*') }} > cache-key
- - name: Cache
- id: cache
+ - name: Get Ccache
uses: actions/cache/restore@v3
with:
- path: |
- ./cpp/build/releases/
- ./cpp/build/velox/udf/examples/
- ./cpp/build/velox/benchmarks/
- /root/.m2/repository/org/apache/arrow/
- key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-centos8-release-default-${{github.sha}}
+ restore-keys: |
+ ccache-centos8-release-default
- name: Setup java and maven
run: |
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
@@ -1056,10 +1061,16 @@ jobs:
yum install sudo patch java-1.8.0-openjdk-devel wget -y
$SETUP install_maven
- name: Build Gluten native libraries
- if: steps.cache.outputs.cache-hit != 'true'
run: |
df -a
bash dev/ci-velox-buildshared-centos-8.sh
+ ccache -s
+ # - name: "Save ccache"
+ # uses: actions/cache/save@v3
+ # id: ccache
+ # with:
+ # path: '${{ env.CCACHE_DIR }}'
+ # key: ccache-centos8-release-default-${{github.sha}}
- name: Run CPP unit test
run: |
cd ./cpp/build && ctest -V
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index c5cbab0697..3e1a91f89f 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -100,7 +100,9 @@ set(KNOWN_WARNINGS
-Wno-error=unused-function \
-Wno-error=unused-variable \
-Wno-strict-aliasing \
- -Wno-ignored-qualifiers")
+ -Wno-ignored-qualifiers \
+ -Wno-deprecated-declarations \
+ -Wno-attributes")
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
set(KNOWN_WARNINGS "-Wno-error=unused-but-set-variable \
@@ -143,8 +145,25 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
add_compile_definitions(_GNU_SOURCE)
endif()
+if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess")
+endif()
+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${KNOWN_WARNINGS}")
+# Keep same compile option with Velox.
+execute_process(
+ COMMAND
+ bash -c
+ "( source ${VELOX_HOME}/scripts/setup-helper-functions.sh && echo -n
$(get_cxx_flags $ENV{CPU_TARGET}))"
+ OUTPUT_VARIABLE SCRIPT_CXX_FLAGS
+ RESULT_VARIABLE COMMAND_STATUS)
+if(COMMAND_STATUS EQUAL "1")
+ message(FATAL_ERROR "Unable to determine compiler flags!")
+endif()
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SCRIPT_CXX_FLAGS}")
+
#
# Dependencies
#
diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt
index cc5b6c7e92..f39bc03bf4 100644
--- a/cpp/core/CMakeLists.txt
+++ b/cpp/core/CMakeLists.txt
@@ -22,10 +22,7 @@ include(FindPkgConfig)
include(GNUInstallDirs)
include(CheckCXXCompilerFlag)
-# Only set arch=native for non-AppleClang compilers.
-if(NOT CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
-endif()
+message("Core module final CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
set(BOOST_MIN_VERSION "1.42.0")
find_package(Boost REQUIRED)
@@ -170,11 +167,6 @@ set_source_files_properties(${GLUTEN_PROTO_OUTPUT_FILES}
PROPERTIES GENERATED
TRUE)
get_filename_component(GLUTEN_PROTO_DIR ${GLUTEN_PROTO_SRC_DIR}/ DIRECTORY)
-set(CMAKE_CXX_FLAGS
- "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-attributes")
-
-message("Core module final CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
-
set(SPARK_COLUMNAR_PLUGIN_SRCS
${SUBSTRAIT_PROTO_SRCS}
${GLUTEN_PROTO_SRCS}
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
index a121b003d7..9a157076d1 100644
--- a/cpp/velox/CMakeLists.txt
+++ b/cpp/velox/CMakeLists.txt
@@ -23,12 +23,6 @@ include(GNUInstallDirs)
include(CheckCXXCompilerFlag)
include(FindPackageHandleStandardArgs)
-set(CMAKE_CXX_FLAGS
- "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-attributes")
-if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess")
-endif()
-
set(SYSTEM_LIB_PATH
"/usr/lib"
CACHE PATH "System Lib dir")
@@ -62,18 +56,6 @@ if(NOT DEFINED VELOX_HOME)
message(STATUS "Set VELOX_HOME to ${VELOX_HOME}")
endif()
-# Keep same compile option with Velox.
-execute_process(
- COMMAND
- bash -c
- "( source ${VELOX_HOME}/scripts/setup-helper-functions.sh && echo -n
$(get_cxx_flags $ENV{CPU_TARGET}))"
- OUTPUT_VARIABLE SCRIPT_CXX_FLAGS
- RESULT_VARIABLE COMMAND_STATUS)
-if(COMMAND_STATUS EQUAL "1")
- message(FATAL_ERROR "Unable to determine compiler flags!")
-endif()
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SCRIPT_CXX_FLAGS}")
-
message("Velox module final CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
# User can specify VELOX_BUILD_PATH, if Velox are built elsewhere.
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 6ba29cf799..67cf9a4746 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -264,7 +264,6 @@ function setup_linux {
local LINUX_VERSION_ID=$(. /etc/os-release && echo ${VERSION_ID})
# apply patches
- sed -i 's/-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++17/-march=native
-std=c++17 -mno-avx512f/g' scripts/setup-helper-functions.sh
sed -i 's/SUDO="${SUDO:-""}"/SUDO="${SUDO:-"sudo --preserve-env"}"/g'
scripts/setup-helper-functions.sh
if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian"
|| "$LINUX_DISTRIBUTION" == "pop" ]]; then
process_setup_ubuntu
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
