This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch hadoop-3.4.0 in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit cf3f82bc5548625480dead2b8f8d54b38ed7b11a Author: Adonis Ling <[email protected]> AuthorDate: Fri Jun 2 13:19:33 2023 +0800 [chore] Build HDFS libraries only (#85) 1. We don't need to build the whole hadoop project. We can build HDFS related libraries to save the build time. 2. Port to GCC-13 --- build.sh | 5 +++-- dev-support/bin/dist-layout-stitching | 18 ++++++++++-------- hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 16 ++++++++++++++-- .../src/main/native/libhdfspp/CMakeLists.txt | 5 ----- .../src/CMakeLists.txt | 1 - 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/build.sh b/build.sh index 717388f3eaf..0cc81fbe3d5 100755 --- a/build.sh +++ b/build.sh @@ -38,9 +38,10 @@ LIBHDFS_DIST_DIR="${SRC_HOME}/hadoop-dist/target/hadoop-libhdfs-3.3.6" rm -rf "${DIST_DIR}" rm -rf "${LIBHDFS_DIST_DIR}" -export PATH=${THIRDPARTY_INSTALLED}/bin:$PATH -mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip=true \ +mvn clean package -Pdist -DskipTests -Dmaven.javadoc.skip -f hadoop-common-project -e +mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip -f hadoop-hdfs-project \ -Dthirdparty.installed="${THIRDPARTY_INSTALLED}" -Dopenssl.prefix="${THIRDPARTY_INSTALLED}" -e +mvn clean package -Pdist -DskipTests -Dmaven.javadoc.skip -f hadoop-dist -e if [[ ! -d "${DIST_DIR}" ]]; then echo "${DIST_DIR} is missing. Build failed." diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching index d4bfd8aaada..4d63d18b238 100755 --- a/dev-support/bin/dist-layout-stitching +++ b/dev-support/bin/dist-layout-stitching @@ -142,14 +142,16 @@ run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" . run cp -pr "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${VERSION}"/* . run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION}"/* . -# copy client jars as-is -run mkdir -p "share/hadoop/client" -run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/ -run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/ -run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/ - -run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" . -run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" . +if ! ${SKIP_HADOOP_CLIENT_AND_TOOLS:-true}; then + # copy client jars as-is + run mkdir -p "share/hadoop/client" + run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/ + run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/ + run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/ + + run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" . + run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" . +fi echo diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml index e9be65d4938..3e7589bae1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml @@ -226,6 +226,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";> <goals><goal>cmake-compile</goal></goals> <configuration> <source>${basedir}/src</source> + <env> + <CXXFLAGS>-include cstdint</CXXFLAGS> + <PROTOBUF_HOME>${thirdparty.installed}</PROTOBUF_HOME> + </env> <vars> <GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH> <JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL> @@ -237,7 +241,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";> <CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX> <CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB> <CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE> - <CUSTOM_THIRDPARTY_INSTALLED>${thirdparty.installed}</CUSTOM_THIRDPARTY_INSTALLED> + <CUSTOM_SNAPPY_PREFIX>${thirdparty.installed}</CUSTOM_SNAPPY_PREFIX> + <HDFSPP_LIBRARY_ONLY>ON</HDFSPP_LIBRARY_ONLY> + <Boost_USE_STATIC_RUNTIME>ON</Boost_USE_STATIC_RUNTIME> </vars> <output>${project.build.directory}</output> </configuration> @@ -294,6 +300,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";> <goals><goal>cmake-compile</goal></goals> <configuration> <source>${basedir}/src</source> + <env> + <CXXFLAGS>-include cstdint</CXXFLAGS> + <PROTOBUF_HOME>${thirdparty.installed}</PROTOBUF_HOME> + </env> <vars> <CMAKE_C_COMPILER>clang</CMAKE_C_COMPILER> <CMAKE_CXX_COMPILER>clang++</CMAKE_CXX_COMPILER> @@ -307,7 +317,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";> <CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX> <CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB> <CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE> - <CUSTOM_THIRDPARTY_INSTALLED>${thirdparty.installed}</CUSTOM_THIRDPARTY_INSTALLED> + <CUSTOM_SNAPPY_PREFIX>${thirdparty.installed}</CUSTOM_SNAPPY_PREFIX> + <HDFSPP_LIBRARY_ONLY>ON</HDFSPP_LIBRARY_ONLY> + <Boost_USE_STATIC_RUNTIME>ON</Boost_USE_STATIC_RUNTIME> </vars> <output>${project.build.directory}/clang</output> </configuration> diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 9fd2ec0e526..a42ee2e2114 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -34,11 +34,6 @@ enable_testing() set(CMAKE_CXX_STANDARD 17) include (CTest) -set(CMAKE_INCLUDE_PATH ${CMAKE_INCLUDE_PATH} "${CUSTOM_THIRDPARTY_INSTALLED}/include") -set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} "${CUSTOM_THIRDPARTY_INSTALLED}/lib") -set(HDFSPP_LIBRARY_ONLY 1) -SET(Boost_USE_STATIC_RUNTIME ON) - SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TRUE'") SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt index 1a0384d50fd..a96290a093b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt @@ -66,7 +66,6 @@ check_function_exists(strtoul HAVE_STRTOUL) # Require snappy. set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES) -set(CUSTOM_SNAPPY_PREFIX "${CUSTOM_THIRDPARTY_INSTALLED}") hadoop_set_find_shared_library_version("1") find_library(SNAPPY_LIBRARY NAMES snappy --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
