HDFS-12110: libhdfs++: Rebase HDFS-8707 on trunk, minor conflict resolution in this patch. Contributed by Deepak Majeti and Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22f91293 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22f91293 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22f91293 Branch: refs/heads/HDFS-12996 Commit: 22f912937b4940829fd6c7ef4e6a6e55228991fa Parents: 5f22a04 Author: James Clampffer <james.clampf...@hp.com> Authored: Fri Feb 16 10:27:11 2018 -0500 Committer: Hanisha Koneru <hanishakon...@apache.org> Committed: Mon Mar 26 11:11:04 2018 -0700 ---------------------------------------------------------------------- dev-support/docker/Dockerfile | 8 ++-- .../hadoop-hdfs-native-client/pom.xml | 44 ++++++++++---------- .../src/CMakeLists.txt | 12 +----- .../src/main/native/libhdfspp/CMakeLists.txt | 5 ++- .../native/libhdfspp/lib/proto/CMakeLists.txt | 1 + .../src/main/native/libhdfspp/tests/hdfs_shim.c | 7 ++++ .../libhdfspp/tests/libhdfs_wrapper_defines.h | 5 +++ .../libhdfspp/tests/libhdfs_wrapper_undefs.h | 5 +++ .../libhdfspp/tests/libhdfspp_wrapper_defines.h | 5 +++ 9 files changed, 54 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/dev-support/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 56db12a..d6bf779 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -38,7 +38,6 @@ ENV DEBCONF_TERSE true # WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default # Ubuntu Java. See Java section below! ###### -<<<<<<< 5f556eacf0fedbaec3cdb819847c7c6c3bacc048 RUN apt-get -q update && apt-get -q install -y \ apt-utils \ build-essential \ @@ -56,6 +55,7 @@ RUN apt-get -q update && apt-get -q install -y \ libfuse-dev \ libprotobuf-dev \ libprotoc-dev \ + libsasl2-dev \ libsnappy-dev \ libssl-dev \ libtool \ @@ -65,7 +65,6 @@ RUN apt-get -q update && apt-get -q install -y \ pkg-config \ python \ python2.7 \ - pylint \ python-pip \ python-pkg-resources \ python-setuptools \ @@ -74,8 +73,8 @@ RUN apt-get -q update && apt-get -q install -y \ software-properties-common \ snappy \ sudo \ - zlib1g-dev \ - valgrind + valgrind \ + zlib1g-dev ####### # OpenJDK 8 @@ -203,3 +202,4 @@ ENV FORREST_HOME=/opt/apache-forrest ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh RUN chmod 755 /root/hadoop_env_checks.sh RUN echo '~/hadoop_env_checks.sh' >> /root/.bashrc + http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml index f596eac..52d9257 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml @@ -31,12 +31,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <properties> <require.fuse>false</require.fuse> - <hadoop.component>hdfs</hadoop.component> <require.libwebhdfs>false</require.libwebhdfs> <require.valgrind>false</require.valgrind> <native_ctest_args></native_ctest_args> <native_cmake_args></native_cmake_args> <native_make_args></native_make_args> + <hadoop.component>hdfs</hadoop.component> </properties> <dependencies> @@ -168,17 +168,21 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <goals><goal>run</goal></goals> <configuration> <skip>${skipTests}</skip> - <target> - <property name="compile_classpath" refid="maven.compile.classpath"/> - <property name="test_classpath" refid="maven.test.classpath"/> - <exec executable="ctest" failonerror="true" dir="${project.build.directory}/"> - <arg line="--output-on-failure"/> - <arg line="${native_ctest_args}"/> - <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> - <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. --> - <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> - </exec> - </target> + <target> + <property name="compile_classpath" refid="maven.compile.classpath"/> + <property name="test_classpath" refid="maven.test.classpath"/> + <exec executable="ctest" failonerror="true" dir="${project.build.directory}/native"> + <arg line="--output-on-failure"/> + <arg line="${native_ctest_args}"/> + <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> + <!-- HADOOP_HOME required to find winutils. --> + <env key="HADOOP_HOME" value="${hadoop.common.build.dir}"/> + <!-- Make sure hadoop.dll and jvm.dll are on PATH. --> + <env key="PATH" value="${env.PATH};${hadoop.common.build.dir}/bin;${java.home}/jre/bin/server;${java.home}/bin/server"/> + <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. --> + <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> + </exec> + </target> </configuration> </execution> </executions> @@ -197,13 +201,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <build> <plugins> <plugin> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-maven-plugins</artifactId> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> <executions> <execution> - <id>cmake-compile</id> + <id>make</id> <phase>compile</phase> - <goals><goal>cmake-compile</goal></goals> + <goals><goal>run</goal></goals> <configuration> <target> <mkdir dir="${project.build.directory}"/> @@ -217,12 +221,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> </target> </configuration> </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> <execution> <id>native_tests</id> <phase>test</phase> @@ -237,7 +235,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <arg line="${native_ctest_args}"/> <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. --> - <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> + <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> </exec> </target> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt index eaec0ec..a3f8f2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt @@ -58,19 +58,11 @@ if(WIN32) # Omit unneeded headers. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN") set(OS_DIR ${CMAKE_SOURCE_DIR}/main/native/libhdfs/os/windows) - - # IMPORTANT: OUT_DIR MUST be relative to maven's - # project.build.directory (=target) and match dist-copynativelibs - # in order to be in a release - set(OUT_DIR bin) + set(OUT_DIR target/bin) else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden") set(OS_DIR ${CMAKE_SOURCE_DIR}/main/native/libhdfs/os/posix) - - # IMPORTANT: OUT_DIR MUST be relative to maven's - # project.build.directory (=target) and match dist-copynativelibs - # in order to be in a release - set(OUT_DIR native/target/usr/local/lib) + set(OUT_DIR target/usr/local/lib) endif() # Configure JNI. http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index e9d87c1..94b1b56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -35,7 +35,10 @@ SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TR SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) # If there's a better way to inform FindCyrusSASL.cmake, let's make this cleaner: -SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR}") +SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR};$ENV{PROTOBUF_HOME}") + +# Specify PROTOBUF_HOME so that find_package picks up the correct version +SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};$ENV{PROTOBUF_HOME}") find_package(Doxygen) find_package(OpenSSL REQUIRED) http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt index a5b5c4c..2eff301 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt @@ -29,6 +29,7 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_HDFS_DIR}/hdfs.proto ${PROTO_HDFS_DIR}/inotify.proto ${PROTO_HDFS_DIR}/xattr.proto + ${PROTO_HDFS_DIR}/ReconfigurationProtocol.proto ${PROTO_HADOOP_DIR}/IpcConnectionContext.proto ${PROTO_HADOOP_DIR}/ProtobufRpcEngine.proto ${PROTO_HADOOP_DIR}/RpcHeader.proto http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c index 1edfedc..54d4cf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c @@ -470,6 +470,13 @@ void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) { return libhdfs_hadoopRzBufferFree(file->libhdfsRep, buffer); } +int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) { + return libhdfs_hdfsGetHedgedReadMetrics(fs->libhdfsRep, (struct libhdfs_hdfsHedgedReadMetrics **) metrics); +} + +void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) { + return libhdfs_hdfsFreeHedgedReadMetrics((struct libhdfs_hdfsHedgedReadMetrics *) metrics); +} /************* * hdfs_ext functions http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h index fcf3185..b907768 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h @@ -90,3 +90,8 @@ #define kObjectKindDirectory libhdfs_kObjectKindDirectory #define hdfsReadStatistics libhdfs_hdfsReadStatistics #define hdfsFileInfo libhdfs_hdfsFileInfo +#define hdfsHedgedReadMetrics libhdfs_hdfsHedgedReadMetrics +#define hdfsGetHedgedReadMetrics libhdfs_hdfsGetHedgedReadMetrics +#define hdfsFreeHedgedReadMetrics libhdfs_hdfsFreeHedgedReadMetrics +#define hdfsStreamBuilderAlloc libhdfs_hdfsStreamBuilderAlloc +#define hdfsStreamBuilderBuild libhdfs_hdfsStreamBuilderBuild http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h index 5a67fce..fce0e82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h @@ -100,3 +100,8 @@ #undef hdfsRenameSnapshot #undef hdfsAllowSnapshot #undef hdfsDisallowSnapshot +#undef hdfsHedgedReadMetrics +#undef hdfsGetHedgedReadMetrics +#undef hdfsFreeHedgedReadMetrics +#undef hdfsStreamBuilderAlloc +#undef hdfsStreamBuilderBuild http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f91293/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h index c9bed90..d0411c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h @@ -100,3 +100,8 @@ #define hdfsRenameSnapshot libhdfspp_hdfsRenameSnapshot #define hdfsAllowSnapshot libhdfspp_hdfsAllowSnapshot #define hdfsDisallowSnapshot libhdfspp_hdfsDisallowSnapshot +#define hdfsHedgedReadMetrics libhdfspp_hdfsHedgedReadMetrics +#define hdfsGetHedgedReadMetrics libhdfspp_hdfsGetHedgedReadMetrics +#define hdfsFreeHedgedReadMetrics libhdfspp_hdfsFreeHedgedReadMetrics +#define hdfsStreamBuilderAlloc libhdfspp_hdfsStreamBuilderAlloc +#define hdfsStreamBuilderBuild libhdfspp_hdfsStreamBuilderBuild --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
