HDFS-13822. speedup libhdfs++ build (enable parallel build). Contributed by Allen Wittenauer and Pradeep Ambati
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a17eed1b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a17eed1b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a17eed1b Branch: refs/heads/HDFS-12943 Commit: a17eed1b870ede9c6519b260e2dfe721b270bdbb Parents: d723285 Author: Jason Lowe <[email protected]> Authored: Fri Aug 17 12:23:18 2018 -0500 Committer: Jason Lowe <[email protected]> Committed: Fri Aug 17 12:25:36 2018 -0500 ---------------------------------------------------------------------- .../hadoop-common/HadoopJNI.cmake | 2 + .../hadoop-hdfs-native-client/pom.xml | 105 ++++++++++--------- .../src/CMakeLists.txt | 48 +++++++++ .../src/main/native/libhdfs/CMakeLists.txt | 6 +- .../main/native/libhdfspp/tests/CMakeLists.txt | 4 + 5 files changed, 114 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a17eed1b/hadoop-common-project/hadoop-common/HadoopJNI.cmake ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/HadoopJNI.cmake b/hadoop-common-project/hadoop-common/HadoopJNI.cmake index 78d7ffd..bf0d73e 100644 --- a/hadoop-common-project/hadoop-common/HadoopJNI.cmake +++ b/hadoop-common-project/hadoop-common/HadoopJNI.cmake @@ -93,5 +93,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") # Otherwise, use the standard FindJNI module to locate the JNI components. # else() + find_package(Java REQUIRED) + include(UseJava) find_package(JNI REQUIRED) endif() http://git-wip-us.apache.org/repos/asf/hadoop/blob/a17eed1b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml index 52d9257..ded1c0d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml @@ -201,26 +201,36 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <build> <plugins> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>make</id> + <id>cmake-compile</id> <phase>compile</phase> - <goals><goal>run</goal></goals> + <goals><goal>cmake-compile</goal></goals> <configuration> - <target> - <mkdir dir="${project.build.directory}"/> - <exec executable="cmake" dir="${project.build.directory}" failonerror="true"> - <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DHADOOP_BUILD=1 -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse} -DREQUIRE_VALGRIND=${require.valgrind} "/> - <arg line="${native_cmake_args}"/> - </exec> - <exec executable="make" dir="${project.build.directory}" failonerror="true"> - <arg line="${native_make_args}"/> - </exec> - </target> + <source>${basedir}/src</source> + <vars> + <GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH> + <JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL> + <REQUIRE_FUSE>${require.fuse}</REQUIRE_FUSE> + <REQUIRE_VALGRIND>${require.valgrind}</REQUIRE_VALGRIND> + <HADOOP_BUILD>1</HADOOP_BUILD> + <REQUIRE_LIBWEBHDFS>${require.libwebhdfs}</REQUIRE_LIBWEBHDFS> + <REQUIRE_OPENSSL>${require.openssl}</REQUIRE_OPENSSL> + <CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX> + <CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB> + <CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE> + </vars> + <output>${project.build.directory}</output> </configuration> </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> <execution> <id>native_tests</id> <phase>test</phase> @@ -236,6 +246,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. --> <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> + <env key="DYLD_LIBRARY_PATH" value="${env.DYLD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> </exec> </target> </configuration> @@ -246,7 +257,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> </build> </profile> <profile> - <id>test-patch</id> + <id>native-clang</id> <activation> <activeByDefault>false</activeByDefault> </activation> @@ -256,35 +267,40 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <build> <plugins> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>make_altern</id> + <id>cmake-compile-clang</id> <phase>compile</phase> - <goals><goal>run</goal></goals> + <goals><goal>cmake-compile</goal></goals> <configuration> - <target> - <mkdir dir="${project.build.directory}/altern"/> - <condition property="c_compiler" value="clang" else="gcc"> - <contains string="${env.CC}" substring="gcc"/> - </condition> - <condition property="cxx_compiler" value="clang++" else="g++"> - <contains string="${env.CXX}" substring="g++"/> - </condition> - <exec executable="cmake" dir="${project.build.directory}/altern" failonerror="true"> - <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/altern/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DHADOOP_BUILD=1 -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse} -DREQUIRE_VALGRIND=${require.valgrind} "/> - <arg line="-DCMAKE_C_COMPILER=${c_compiler} -DCMAKE_CXX_COMPILER=${cxx_compiler}"/> - <arg line="${native_cmake_args}"/> - </exec> - <exec executable="make" dir="${project.build.directory}/altern" failonerror="true"> - <arg line="${native_make_args}"/> - </exec> - </target> + <source>${basedir}/src</source> + <vars> + <CMAKE_C_COMPILER>clang</CMAKE_C_COMPILER> + <CMAKE_CXX_COMPILER>clang++</CMAKE_CXX_COMPILER> + <GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH> + <JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL> + <REQUIRE_FUSE>${require.fuse}</REQUIRE_FUSE> + <REQUIRE_VALGRIND>${require.valgrind}</REQUIRE_VALGRIND> + <HADOOP_BUILD>1</HADOOP_BUILD> + <REQUIRE_LIBWEBHDFS>${require.libwebhdfs}</REQUIRE_LIBWEBHDFS> + <REQUIRE_OPENSSL>${require.openssl}</REQUIRE_OPENSSL> + <CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX> + <CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB> + <CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE> + </vars> + <output>${project.build.directory}/clang</output> </configuration> </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> <execution> - <id>native_tests_altern</id> + <id>native_tests_clang</id> <phase>test</phase> <goals><goal>run</goal></goals> <configuration> @@ -292,26 +308,17 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <target> <property name="compile_classpath" refid="maven.compile.classpath"/> <property name="test_classpath" refid="maven.test.classpath"/> - <exec executable="ctest" failonerror="true" dir="${project.build.directory}/altern"> + <exec executable="ctest" failonerror="true" dir="${project.build.directory}/clang"> <arg line="--output-on-failure"/> <arg line="${native_ctest_args}"/> <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. --> - <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/altern/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> + <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/clang/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> + <env key="DYLD_LIBRARY_PATH" value="${env.DYLD_LIBRARY_PATH}:${project.build.directory}/clang/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> </exec> </target> </configuration> </execution> - <execution> - <id>clean_altern</id> - <phase>test</phase> - <goals><goal>run</goal></goals> - <configuration> - <target> - <delete dir="${project.build.directory}/altern" includeemptydirs="true"/> - </target> - </configuration> - </execution> </executions> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/hadoop/blob/a17eed1b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt index a3f8f2d..1813ec1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt @@ -88,6 +88,54 @@ function(link_libhdfs_test NAME LIBRARY) target_link_libraries("${NAME}_${LIBRARY}" ${LIBRARY} ${ARGN}) endfunction() + +set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) +hadoop_set_find_shared_library_without_version() +set(OPENSSL_NAME "crypto") +if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + SET(OPENSSL_NAME "eay32") +endif() +message("CUSTOM_OPENSSL_PREFIX = ${CUSTOM_OPENSSL_PREFIX}") +find_library(OPENSSL_LIBRARY + NAMES ${OPENSSL_NAME} + PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/lib + ${CUSTOM_OPENSSL_PREFIX}/lib64 ${CUSTOM_OPENSSL_LIB} NO_DEFAULT_PATH) +find_library(OPENSSL_LIBRARY NAMES ${OPENSSL_NAME}) +find_path(OPENSSL_INCLUDE_DIR + NAMES openssl/evp.h + PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/include + ${CUSTOM_OPENSSL_INCLUDE} NO_DEFAULT_PATH) +find_path(OPENSSL_INCLUDE_DIR NAMES openssl/evp.h) +set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) +set(USABLE_OPENSSL 0) +if(OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR) + include(CheckCSourceCompiles) + set(OLD_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES}) + set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR}) + check_c_source_compiles("#include \"${OPENSSL_INCLUDE_DIR}/openssl/evp.h\"\nint main(int argc, char **argv) { return !EVP_aes_256_ctr; }" HAS_NEW_ENOUGH_OPENSSL) + set(CMAKE_REQUIRED_INCLUDES ${OLD_CMAKE_REQUIRED_INCLUDES}) + if(NOT HAS_NEW_ENOUGH_OPENSSL) + message("The OpenSSL library installed at ${OPENSSL_LIBRARY} is too old. You need a version at least new enough to have EVP_aes_256_ctr.") + else() + SET(USABLE_OPENSSL 1) + endif() +endif() +if(USABLE_OPENSSL) + get_filename_component(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME) + set(OPENSSL_SOURCE_FILES + "${SRC}/crypto/OpensslCipher.c" + "${SRC}/crypto/random/OpensslSecureRandom.c") + set(REQUIRE_OPENSSL ${REQUIRE_OPENSSL}) # Stop warning about unused variable. +else() + message("Cannot find a usable OpenSSL library. OPENSSL_LIBRARY=${OPENSSL_LIBRARY}, OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_LIB=${CUSTOM_OPENSSL_LIB}, CUSTOM_OPENSSL_PREFIX=${CUSTOM_OPENSSL_PREFIX}, CUSTOM_OPENSSL_INCLUDE=${CUSTOM_OPENSSL_INCLUDE}") + if(REQUIRE_OPENSSL) + message(FATAL_ERROR "Terminating build because require.openssl was specified.") + endif() + set(OPENSSL_LIBRARY "") + set(OPENSSL_INCLUDE_DIR "") + set(OPENSSL_SOURCE_FILES "") +endif() + add_subdirectory(main/native/libhdfs) add_subdirectory(main/native/libhdfs-tests) add_subdirectory(main/native/libhdfspp) http://git-wip-us.apache.org/repos/asf/hadoop/blob/a17eed1b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt index 2883585..cac1335 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt @@ -74,9 +74,11 @@ endif() # introducing an abstraction layer over the sys/mman.h functions. if(NOT WIN32) build_libhdfs_test(test_libhdfs_vecsum hdfs vecsum.c) + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) if(CMAKE_SYSTEM_NAME MATCHES "Darwin") - link_libhdfs_test(test_libhdfs_vecsum hdfs pthread) + link_libhdfs_test(test_libhdfs_vecsum hdfs ${CMAKE_THREAD_LIBS_INIT}) else() - link_libhdfs_test(test_libhdfs_vecsum hdfs pthread rt) + link_libhdfs_test(test_libhdfs_vecsum hdfs ${CMAKE_THREAD_LIBS_INIT} rt) endif() endif() http://git-wip-us.apache.org/repos/asf/hadoop/blob/a17eed1b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 6157902..59fdbf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -143,6 +143,10 @@ include_directories ( add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc) add_library(hdfspp_test_static STATIC ${LIBHDFSPP_BINDING_C}/hdfs.cc) +# Add dependencies +add_dependencies(hdfspp_test_shim_static proto) +add_dependencies(hdfspp_test_static proto) + # TODO: get all of the mini dfs library bits here in one place # add_library(hdfspp_mini_cluster native_mini_dfs ${JAVA_JVM_LIBRARY} ) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
