This is an automated email from the ASF dual-hosted git repository.
weitingchen pushed a commit to branch branch-1.2
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/branch-1.2 by this push:
new be17f7eeaa [VL][1.2] Port #6560 #6569 #6730 #7117 for vcpkg issue fix
(#7974)
be17f7eeaa is described below
commit be17f7eeaa1628ed8fdff4fb27ea4ae6d45416c7
Author: Wei-Ting Chen <[email protected]>
AuthorDate: Wed Nov 20 15:45:40 2024 +0800
[VL][1.2] Port #6560 #6569 #6730 #7117 for vcpkg issue fix (#7974)
* [MINOR] Remove duplicate mvn packages (#6560)
* [VL] Improve package scripts (#6569)
* [GLUTEN-6569][VL] Delete unnecessary gcc9 enabling from package script
(#6730)
* [GLUTEN-7118][VL] Fix duckdb target issue when vcpkg is enabled (#7117)
---------
Co-authored-by: Zhen Wang <[email protected]>
Co-authored-by: PHILO-HE <[email protected]>
---
dev/{package.sh => build-thirdparty.sh} | 17 +++----
dev/package-vcpkg.sh | 18 +++++---
dev/package.sh | 78 +++------------------------------
dev/vcpkg/Makefile | 2 +-
dev/vcpkg/ports/duckdb/portfile.cmake | 7 +++
dev/vcpkg/ports/duckdb/vcpkg.json | 9 +++-
6 files changed, 43 insertions(+), 88 deletions(-)
diff --git a/dev/package.sh b/dev/build-thirdparty.sh
similarity index 87%
copy from dev/package.sh
copy to dev/build-thirdparty.sh
index 7e7e793bda..90fc5e58fb 100755
--- a/dev/package.sh
+++ b/dev/build-thirdparty.sh
@@ -9,13 +9,6 @@ LINUX_OS=$(. /etc/os-release && echo ${ID})
VERSION=$(. /etc/os-release && echo ${VERSION_ID})
ARCH=`uname -m`
-# compile gluten jar
-$GLUTEN_DIR/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON
--enable_s3=ON --enable_hdfs=ON
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.2 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.3 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.4 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.5 -DskipTests
-
mkdir -p $THIRDPARTY_LIB
function process_setup_ubuntu_2004 {
cp
/usr/lib/${ARCH}-linux-gnu/{libroken.so.18,libasn1.so.8,libcrypto.so.1.1,libnghttp2.so.14,libnettle.so.7,libhogweed.so.5,librtmp.so.1,libssh.so.4,libssl.so.1.1,liblber-2.4.so.2,libsasl2.so.2,libwind.so.0,libheimbase.so.1,libhcrypto.so.4,libhx509.so.5,libkrb5.so.26,libheimntlm.so.0,libgssapi.so.3,libldap_r-2.4.so.2,libcurl.so.4,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libunwind.so.8,libglog.so.0,libidn.so.11,libntlm.so.0,libgsasl.so.7,libicudata.so.66,libicuuc.so.
[...]
@@ -27,6 +20,12 @@ function process_setup_ubuntu_2204 {
cp
/usr/local/lib/{libhdfs3.so.1,libprotobuf.so.32,libboost_context.so.1.84.0,libboost_regex.so.1.84.0}
$THIRDPARTY_LIB/
}
+function process_setup_centos_9 {
+ cp
/lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libgsasl.so.7,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libidn.so.12,libntlm.so.0,libsodium.so.23}
$THIRDPARTY_LIB/
+ cp
/usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32}
$THIRDPARTY_LIB/
+ cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1} $THIRDPARTY_LIB/
+}
+
function process_setup_centos_8 {
cp
/usr/lib64/{libre2.so.0,libdouble-conversion.so.3,libevent-2.1.so.6,libdwarf.so.1,libgsasl.so.7,libicudata.so.60,libicui18n.so.60,libicuuc.so.60,libidn.so.11,libntlm.so.0,libsodium.so.23}
$THIRDPARTY_LIB/
cp
/usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32}
$THIRDPARTY_LIB/
@@ -56,7 +55,9 @@ if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then
process_setup_ubuntu_2204
fi
elif [ "$LINUX_OS" == "centos" ]; then
- if [ "$VERSION" == "8" ]; then
+ if [ "$VERSION" == "9" ]; then
+ process_setup_centos_9
+ elif [ "$VERSION" == "8" ]; then
process_setup_centos_8
elif [ "$VERSION" == "7" ]; then
process_setup_centos_7
diff --git a/dev/package-vcpkg.sh b/dev/package-vcpkg.sh
index c76b8be8d7..8f2b219f0d 100755
--- a/dev/package-vcpkg.sh
+++ b/dev/package-vcpkg.sh
@@ -1,21 +1,27 @@
#!/bin/bash
-set -ex
+set -eux
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
GLUTEN_DIR="$CURRENT_DIR/.."
+LINUX_OS=$(. /etc/os-release && echo ${ID})
+VERSION=$(. /etc/os-release && echo ${VERSION_ID})
+ARCH=`uname -m`
cd "$GLUTEN_DIR"
if [ "$LINUX_OS" == "centos" ]; then
if [ "$VERSION" == "8" ]; then
source /opt/rh/gcc-toolset-9/enable
elif [ "$VERSION" == "7" ]; then
+ export MANPATH=""
source /opt/rh/devtoolset-9/enable
fi
fi
+
+# prepare vcpkg environments
source ./dev/vcpkg/env.sh
-./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON
--enable_s3=ON --enable_hdfs=ON
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.2 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.3 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.4 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.5 -DskipTests
+
+# build gluten with velox backend, prompt always respond y
+export PROMPT_ALWAYS_RESPOND=y
+
+./dev/buildbundle-veloxbe.sh --build_tests=ON --build_arrow=OFF
--build_benchmarks=ON --enable_s3=ON --enable_hdfs=ON "$@"
diff --git a/dev/package.sh b/dev/package.sh
index 7e7e793bda..a26f94d9b6 100755
--- a/dev/package.sh
+++ b/dev/package.sh
@@ -4,81 +4,15 @@ set -eux
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
GLUTEN_DIR="$CURRENT_DIR/.."
-THIRDPARTY_LIB="$GLUTEN_DIR/package/target/thirdparty-lib"
LINUX_OS=$(. /etc/os-release && echo ${ID})
VERSION=$(. /etc/os-release && echo ${VERSION_ID})
ARCH=`uname -m`
-# compile gluten jar
-$GLUTEN_DIR/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON
--enable_s3=ON --enable_hdfs=ON
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.2 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.3 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.4 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.5 -DskipTests
+cd "$GLUTEN_DIR"
-mkdir -p $THIRDPARTY_LIB
-function process_setup_ubuntu_2004 {
- cp
/usr/lib/${ARCH}-linux-gnu/{libroken.so.18,libasn1.so.8,libcrypto.so.1.1,libnghttp2.so.14,libnettle.so.7,libhogweed.so.5,librtmp.so.1,libssh.so.4,libssl.so.1.1,liblber-2.4.so.2,libsasl2.so.2,libwind.so.0,libheimbase.so.1,libhcrypto.so.4,libhx509.so.5,libkrb5.so.26,libheimntlm.so.0,libgssapi.so.3,libldap_r-2.4.so.2,libcurl.so.4,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libunwind.so.8,libglog.so.0,libidn.so.11,libntlm.so.0,libgsasl.so.7,libicudata.so.66,libicuuc.so.
[...]
- cp
/usr/local/lib/{libprotobuf.so.32,libhdfs3.so.1,libboost_context.so.1.84.0,libboost_regex.so.1.84.0}
$THIRDPARTY_LIB/
-}
+# build gluten with velox backend, prompt always respond y
+export PROMPT_ALWAYS_RESPOND=y
+./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON
--enable_s3=ON --enable_hdfs=ON "$@"
-function process_setup_ubuntu_2204 {
- cp
/usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libidn.so.12,libglog.so.0,libgflags.so.2.2,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libgsasl.so.7,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,libntlm.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70,libthrift-0.16.0.so}
$THIRDPARTY_LIB/
- cp
/usr/local/lib/{libhdfs3.so.1,libprotobuf.so.32,libboost_context.so.1.84.0,libboost_regex.so.1.84.0}
$THIRDPARTY_LIB/
-}
-
-function process_setup_centos_8 {
- cp
/usr/lib64/{libre2.so.0,libdouble-conversion.so.3,libevent-2.1.so.6,libdwarf.so.1,libgsasl.so.7,libicudata.so.60,libicui18n.so.60,libicuuc.so.60,libidn.so.11,libntlm.so.0,libsodium.so.23}
$THIRDPARTY_LIB/
- cp
/usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32}
$THIRDPARTY_LIB/
- cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1} $THIRDPARTY_LIB/
-}
-
-function process_setup_centos_7 {
- cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.0} $THIRDPARTY_LIB/
- cp
/usr/lib64/{libdouble-conversion.so.1,libevent-2.0.so.5,libzstd.so.1,libntlm.so.0,libgsasl.so.7,liblz4.so.1}
$THIRDPARTY_LIB/
- cp
/usr/local/lib/{libre2.so.10,libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_regex.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32}
$THIRDPARTY_LIB/
-}
-
-function process_setup_debian_11 {
- cp
/usr/lib/x86_64-linux-gnu/{libre2.so.9,libthrift-0.13.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libglog.so.0,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.67,libicuuc.so.67,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap_r-2.4.so.2,liblber-2.4.so.2,libbrotlidec.so.1,libicudata.so.67,libsasl2.so.2,libbrotlicommon.so.1}
$THIRDPARTY_LIB/
- cp
/usr/local/lib/{libhdfs3.so.1,libprotobuf.so.32,libboost_context.so.1.84.0,libboost_regex.so.1.84.0}
$THIRDPARTY_LIB/
-}
-
-function process_setup_debian_12 {
- cp
/usr/lib/x86_64-linux-gnu/{libthrift-0.17.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libglog.so.1,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.72,libicuuc.so.72,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap-2.5.so.0,liblber-2.5.so.0,libbrotlidec.so.1,libicudata.so.72,libsasl2.so.2,libbrotlicommon.so.1,libcrypto.so.3,libssl.so.3,libgssapi_krb5.so.2,libkrb5.so.3,libk5crypto.so.3,libkrb5support.so.0,libkeyutils.so.1}
$THIRDPARTY_LIB/
- cp
/usr/local/lib/{libprotobuf.so.32,libhdfs3.so.1,libboost_context.so.1.84.0,libboost_regex.so.1.84.0}
$THIRDPARTY_LIB/
-}
-
-if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then
- if [ "$VERSION" == "20.04" ]; then
- process_setup_ubuntu_2004
- elif [ "$VERSION" == "22.04" ]; then
- process_setup_ubuntu_2204
- fi
-elif [ "$LINUX_OS" == "centos" ]; then
- if [ "$VERSION" == "8" ]; then
- process_setup_centos_8
- elif [ "$VERSION" == "7" ]; then
- process_setup_centos_7
- fi
-elif [ "$LINUX_OS" == "alinux" ]; then
- if [ "${VERSION:0:1}" == "3" ]; then
- process_setup_centos_8
- elif [ "${VERSION:0:1}" == "2" ]; then
- process_setup_centos_7
- fi
-elif [ "$LINUX_OS" == "tencentos" ]; then
- if [ "$VERSION" == "2.4" ]; then
- process_setup_centos_7
- elif [ "$VERSION" == "3.2" ]; then
- process_setup_centos_8
- fi
-elif [ "$LINUX_OS" == "debian" ]; then
- if [ "$VERSION" == "11" ]; then
- process_setup_debian_11
- elif [ "$VERSION" == "12" ]; then
- process_setup_debian_12
- fi
-fi
-cd $THIRDPARTY_LIB/
-jar cvf gluten-thirdparty-lib-$LINUX_OS-$VERSION-$ARCH.jar ./
+# make thirdparty package
+./dev/build-thirdparty.sh
diff --git a/dev/vcpkg/Makefile b/dev/vcpkg/Makefile
index 11cd02d881..cb340f95b2 100644
--- a/dev/vcpkg/Makefile
+++ b/dev/vcpkg/Makefile
@@ -1,4 +1,4 @@
-DOCKER_IMAGE=apache/gluten:gluten-vcpkg-builder_2024_05_22
+DOCKER_IMAGE=apache/gluten:vcpkg-centos-7
GLUTEN_REPO=$(shell realpath -L ../..)
CCACHE_DIR=$(HOME)/.ccache
diff --git a/dev/vcpkg/ports/duckdb/portfile.cmake
b/dev/vcpkg/ports/duckdb/portfile.cmake
index e7806f1e4f..a572292953 100644
--- a/dev/vcpkg/ports/duckdb/portfile.cmake
+++ b/dev/vcpkg/ports/duckdb/portfile.cmake
@@ -22,4 +22,11 @@ vcpkg_configure_cmake(
)
vcpkg_install_cmake(ADD_BIN_TO_PATH)
+vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/DuckDB)
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/lib/cmake")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/lib/cmake")
+
+vcpkg_cmake_config_fixup(PACKAGE_NAME DuckDB)
+
vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE")
diff --git a/dev/vcpkg/ports/duckdb/vcpkg.json
b/dev/vcpkg/ports/duckdb/vcpkg.json
index 511bea530f..24f32f7c73 100644
--- a/dev/vcpkg/ports/duckdb/vcpkg.json
+++ b/dev/vcpkg/ports/duckdb/vcpkg.json
@@ -2,6 +2,13 @@
"name": "duckdb",
"version": "0.8.1",
"dependencies": [
-
+ {
+ "name": "vcpkg-cmake",
+ "host": true
+ },
+ {
+ "name": "vcpkg-cmake-config",
+ "host": true
+ }
]
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
