Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/pom.xml Tue Aug 19 23:49:39 2014 @@ -122,11 +122,6 @@ http://maven.apache.org/xsd/maven-4.0.0. <scope>compile</scope> </dependency> <dependency> - <groupId>javax.servlet.jsp</groupId> - <artifactId>jsp-api</artifactId> - <scope>compile</scope> - </dependency> - <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <scope>compile</scope> @@ -147,6 +142,11 @@ http://maven.apache.org/xsd/maven-4.0.0. <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minikdc</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> @@ -167,11 +167,6 @@ http://maven.apache.org/xsd/maven-4.0.0. <scope>compile</scope> </dependency> <dependency> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - <scope>compile</scope> - </dependency> - <dependency> <groupId>xmlenc</groupId> <artifactId>xmlenc</artifactId> <scope>compile</scope> @@ -181,6 +176,11 @@ http://maven.apache.org/xsd/maven-4.0.0. <artifactId>netty</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + <scope>compile</scope> + </dependency> </dependencies> <build> @@ -202,123 +202,6 @@ http://maven.apache.org/xsd/maven-4.0.0. </configuration> </plugin> <plugin> - <groupId>org.codehaus.mojo.jspc</groupId> - <artifactId>jspc-maven-plugin</artifactId> - <executions> - <execution> - <id>hdfs</id> - <phase>generate-sources</phase> - <goals> - <goal>compile</goal> - </goals> - <configuration> - <compile>false</compile> - <workingDirectory>${project.build.directory}/generated-sources/java</workingDirectory> - <webFragmentFile>${project.build.directory}/hdfs-jsp-servlet-definitions.xml</webFragmentFile> - <packageName>org.apache.hadoop.hdfs.server.namenode</packageName> - <sources> - <directory>${basedir}/src/main/webapps/hdfs</directory> - <includes> - <include>*.jsp</include> - </includes> - </sources> - </configuration> - </execution> - <execution> - <id>secondary</id> - <phase>generate-sources</phase> - <goals> - <goal>compile</goal> - </goals> - <configuration> - <compile>false</compile> - <workingDirectory>${project.build.directory}/generated-sources/java</workingDirectory> - <webFragmentFile>${project.build.directory}/secondary-jsp-servlet-definitions.xml</webFragmentFile> - <packageName>org.apache.hadoop.hdfs.server.namenode</packageName> - <sources> - <directory>${basedir}/src/main/webapps/secondary</directory> - <includes> - <include>*.jsp</include> - </includes> - </sources> - </configuration> - </execution> - <execution> - <id>journal</id> - <phase>generate-sources</phase> - <goals> - <goal>compile</goal> - </goals> - <configuration> - <compile>false</compile> - <workingDirectory>${project.build.directory}/generated-sources/java</workingDirectory> - <webFragmentFile>${project.build.directory}/journal-jsp-servlet-definitions.xml</webFragmentFile> - <packageName>org.apache.hadoop.hdfs.server.journalservice</packageName> - <sources> - <directory>${basedir}/src/main/webapps/journal</directory> - <includes> - <include>*.jsp</include> - </includes> - </sources> - </configuration> - </execution> - <execution> - <id>datanode</id> - <phase>generate-sources</phase> - <goals> - <goal>compile</goal> - </goals> - <configuration> - <compile>false</compile> - <workingDirectory>${project.build.directory}/generated-sources/java</workingDirectory> - <webFragmentFile>${project.build.directory}/datanode-jsp-servlet-definitions.xml</webFragmentFile> - <packageName>org.apache.hadoop.hdfs.server.datanode</packageName> - <sources> - <directory>${basedir}/src/main/webapps/datanode</directory> - <includes> - <include>*.jsp</include> - </includes> - </sources> - </configuration> - </execution> - </executions> - <dependencies> - <dependency> - <groupId>org.codehaus.mojo.jspc</groupId> - <artifactId>jspc-compiler-tomcat5</artifactId> - <version>2.0-alpha-3</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>1.4.1</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jcl104-over-slf4j</artifactId> - <version>1.4.1</version> - </dependency> - </dependencies> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>add-jsp-generated-sources-directory</id> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>${project.build.directory}/generated-sources/java</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> <configuration> @@ -326,18 +209,6 @@ http://maven.apache.org/xsd/maven-4.0.0. </configuration> <executions> <execution> - <id>create-jsp-generated-sources-directory</id> - <phase>initialize</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target> - <mkdir dir="${project.build.directory}/generated-sources/java" /> - </target> - </configuration> - </execution> - <execution> <id>create-web-xmls</id> <phase>compile</phase> <goals> @@ -345,32 +216,21 @@ http://maven.apache.org/xsd/maven-4.0.0. </goals> <configuration> <target> - <loadfile property="hdfs.servlet.definitions" srcFile="${project.build.directory}/hdfs-jsp-servlet-definitions.xml"/> - <loadfile property="secondary.servlet.definitions" srcFile="${project.build.directory}/secondary-jsp-servlet-definitions.xml"/> - <loadfile property="datanode.servlet.definitions" srcFile="${project.build.directory}/datanode-jsp-servlet-definitions.xml"/> - <loadfile property="journal.servlet.definitions" srcFile="${project.build.directory}/journal-jsp-servlet-definitions.xml"/> - <echoproperties destfile="${project.build.directory}/webxml.properties"> - <propertyset> - <propertyref regex=".*.servlet.definitions"/> - </propertyset> - </echoproperties> - <filter filtersfile="${project.build.directory}/webxml.properties"/> - <copy file="${basedir}/src/main/webapps/proto-hdfs-web.xml" + <copy file="${basedir}/src/main/webapps/proto-web.xml" tofile="${project.build.directory}/webapps/hdfs/WEB-INF/web.xml" filtering="true"/> - <copy file="${basedir}/src/main/webapps/proto-secondary-web.xml" + <copy file="${basedir}/src/main/webapps/proto-web.xml" tofile="${project.build.directory}/webapps/secondary/WEB-INF/web.xml" filtering="true"/> - <copy file="${basedir}/src/main/webapps/proto-datanode-web.xml" + <copy file="${basedir}/src/main/webapps/proto-web.xml" tofile="${project.build.directory}/webapps/datanode/WEB-INF/web.xml" filtering="true"/> - <copy file="${basedir}/src/main/webapps/proto-journal-web.xml" + <copy file="${basedir}/src/main/webapps/proto-web.xml" tofile="${project.build.directory}/webapps/journal/WEB-INF/web.xml" filtering="true"/> <copy toDir="${project.build.directory}/webapps"> <fileset dir="${basedir}/src/main/webapps"> - <exclude name="**/*.jsp"/> - <exclude name="**/proto-*-web.xml"/> + <exclude name="**/proto-web.xml"/> </fileset> </copy> </target> @@ -391,6 +251,7 @@ http://maven.apache.org/xsd/maven-4.0.0. <copy todir="${project.build.directory}/test-classes/webapps"> <fileset dir="${project.build.directory}/webapps"> <exclude name="proto-*-web.xml"/> + <exclude name="**/proto-web.xml"/> </fileset> </copy> </target> @@ -439,6 +300,7 @@ http://maven.apache.org/xsd/maven-4.0.0. <include>NamenodeProtocol.proto</include> <include>QJournalProtocol.proto</include> <include>acl.proto</include> + <include>xattr.proto</include> <include>datatransfer.proto</include> <include>fsimage.proto</include> <include>hdfs.proto</include> @@ -498,16 +360,97 @@ http://maven.apache.org/xsd/maven-4.0.0. <profiles> <profile> - <id>windows</id> + <id>native-win</id> <activation> <activeByDefault>false</activeByDefault> <os> <family>windows</family> </os> </activation> - <properties> - <windows.build>true</windows.build> - </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-os</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireOS> + <family>windows</family> + <message>native-win build only supported on Windows</message> + </requireOS> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>make</id> + <phase>compile</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <mkdir dir="${project.build.directory}/native"/> + <exec executable="cmake" dir="${project.build.directory}/native" + failonerror="true"> + <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse} -G 'Visual Studio 10 Win64'"/> + </exec> + <exec executable="msbuild" dir="${project.build.directory}/native" + failonerror="true"> + <arg line="ALL_BUILD.vcxproj /nologo /p:Configuration=Release"/> + </exec> + <!-- Copy for inclusion in distribution. --> + <copy todir="${project.build.directory}/bin"> + <fileset dir="${project.build.directory}/native/target/bin/Release"/> + </copy> + </target> + </configuration> + </execution> + <execution> + <id>native_tests</id> + <phase>test</phase> + <goals><goal>run</goal></goals> + <configuration> + <skip>${skipTests}</skip> + <target> + <property name="compile_classpath" refid="maven.compile.classpath"/> + <property name="test_classpath" refid="maven.test.classpath"/> + <macrodef name="run-test"> + <attribute name="test"/> + <sequential> + <echo message="Running @{test}"/> + <exec executable="${project.build.directory}/native/Release/@{test}" failonerror="true" dir="${project.build.directory}/native/"> + <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> + <!-- HADOOP_HOME required to find winutils. --> + <env key="HADOOP_HOME" value="${hadoop.common.build.dir}"/> + <!-- Make sure hadoop.dll and jvm.dll are on PATH. --> + <env key="PATH" value="${env.PATH};${hadoop.common.build.dir}/bin;${java.home}/jre/bin/server;${java.home}/bin/server"/> + </exec> + <echo message="Finished @{test}"/> + </sequential> + </macrodef> + <run-test test="test_libhdfs_threaded"/> + <echo message="Skipping test_libhdfs_zerocopy"/> + <run-test test="test_native_mini_dfs"/> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> </profile> <profile> <id>native</id> @@ -545,21 +488,25 @@ http://maven.apache.org/xsd/maven-4.0.0. <phase>test</phase> <goals><goal>run</goal></goals> <configuration> + <skip>${skipTests}</skip> <target> <property name="compile_classpath" refid="maven.compile.classpath"/> <property name="test_classpath" refid="maven.test.classpath"/> - <exec executable="sh" failonerror="true" dir="${project.build.directory}/native/"> - <arg value="-c"/> - <arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_libhdfs_threaded"/> - <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> - <env key="SKIPTESTS" value="${skipTests}"/> - </exec> - <exec executable="sh" failonerror="true" dir="${project.build.directory}/native/"> - <arg value="-c"/> - <arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_libhdfs_threaded"/> - <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> - <env key="SKIPTESTS" value="${skipTests}"/> - </exec> + <macrodef name="run-test"> + <attribute name="test"/> + <sequential> + <echo message="Running @{test}"/> + <exec executable="${project.build.directory}/native/@{test}" failonerror="true" dir="${project.build.directory}/native/"> + <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/> + <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. --> + <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/> + </exec> + <echo message="Finished @{test}"/> + </sequential> + </macrodef> + <run-test test="test_libhdfs_threaded"/> + <run-test test="test_libhdfs_zerocopy"/> + <run-test test="test_native_mini_dfs"/> </target> </configuration> </execution>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt Tue Aug 19 23:49:39 2014 @@ -62,6 +62,9 @@ endfunction() INCLUDE(CheckCSourceCompiles) CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS) +# Check to see if we have Intel SSE intrinsics. +CHECK_C_SOURCE_COMPILES("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS) + # Check if we need to link dl library to get dlopen. # dlopen on Linux is in separate library but on FreeBSD its in libc INCLUDE(CheckLibraryExists) @@ -73,9 +76,39 @@ if (NOT GENERATED_JAVAH) MESSAGE(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH") endif (NOT GENERATED_JAVAH) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64") +if (WIN32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2") + + # Set warning level 4. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4") + + # Skip "unreferenced formal parameter". + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4100") + + # Skip "conditional expression is constant". + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127") + + # Skip deprecated POSIX function warnings. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_NONSTDC_NO_DEPRECATE") + + # Skip CRT non-secure function warnings. If we can convert usage of + # strerror, getenv and ctime to their secure CRT equivalents, then we can + # re-enable the CRT non-secure function warnings. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS") + + # Omit unneeded headers. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN") + + set(OS_DIR main/native/libhdfs/os/windows) + set(OUT_DIR target/bin) +else (WIN32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64") + set(OS_DIR main/native/libhdfs/os/posix) + set(OS_LINK_LIBRARIES pthread) + set(OUT_DIR target/usr/local/lib) +endif (WIN32) include_directories( ${GENERATED_JAVAH} @@ -84,6 +117,7 @@ include_directories( ${JNI_INCLUDE_DIRS} main/native main/native/libhdfs + ${OS_DIR} ) set(_FUSE_DFS_VERSION 0.1.0) @@ -93,6 +127,9 @@ add_dual_library(hdfs main/native/libhdfs/exception.c main/native/libhdfs/jni_helper.c main/native/libhdfs/hdfs.c + main/native/libhdfs/common/htable.c + ${OS_DIR}/mutexes.c + ${OS_DIR}/thread_local_storage.c ) if (NEED_LINK_DL) set(LIB_DL dl) @@ -101,17 +138,14 @@ endif(NEED_LINK_DL) target_link_dual_libraries(hdfs ${JAVA_JVM_LIBRARY} ${LIB_DL} - pthread + ${OS_LINK_LIBRARIES} ) -dual_output_directory(hdfs target/usr/local/lib) + +dual_output_directory(hdfs ${OUT_DIR}) set(LIBHDFS_VERSION "0.0.0") set_target_properties(hdfs PROPERTIES SOVERSION ${LIBHDFS_VERSION}) -add_library(posix_util - main/native/util/posix_util.c -) - add_executable(test_libhdfs_ops main/native/libhdfs/test/test_libhdfs_ops.c ) @@ -153,11 +187,12 @@ target_link_libraries(test_native_mini_d add_executable(test_libhdfs_threaded main/native/libhdfs/expect.c main/native/libhdfs/test_libhdfs_threaded.c + ${OS_DIR}/thread.c ) target_link_libraries(test_libhdfs_threaded hdfs native_mini_dfs - pthread + ${OS_LINK_LIBRARIES} ) add_executable(test_libhdfs_zerocopy @@ -167,9 +202,22 @@ add_executable(test_libhdfs_zerocopy target_link_libraries(test_libhdfs_zerocopy hdfs native_mini_dfs - pthread + ${OS_LINK_LIBRARIES} ) +# Skip vecsum on Windows. This could be made to work in the future by +# introducing an abstraction layer over the sys/mman.h functions. +if (NOT WIN32) + add_executable(test_libhdfs_vecsum + main/native/libhdfs/test/vecsum.c + ) + target_link_libraries(test_libhdfs_vecsum + hdfs + pthread + rt + ) +endif(NOT WIN32) + IF(REQUIRE_LIBWEBHDFS) add_subdirectory(contrib/libwebhdfs) ENDIF(REQUIRE_LIBWEBHDFS) Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake Tue Aug 19 23:49:39 2014 @@ -22,4 +22,6 @@ #cmakedefine HAVE_BETTER_TLS +#cmakedefine HAVE_INTEL_SSE_INTRINSICS + #endif Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml Tue Aug 19 23:49:39 2014 @@ -163,38 +163,24 @@ http://maven.apache.org/xsd/maven-4.0.0. <build> <plugins> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <version>1.5</version> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.8</version> <executions> <execution> + <id>dist</id> <phase>package</phase> <goals> - <goal>shade</goal> + <goal>copy</goal> </goals> <configuration> - <createDependencyReducedPom>false</createDependencyReducedPom> - <artifactSet> - <includes> - <include>org.apache.bookkeeper:bookkeeper-server</include> - <include>org.apache.zookeeper:zookeeper</include> - <include>org.jboss.netty:netty</include> - </includes> - </artifactSet> - <relocations> - <relocation> - <pattern>org.apache.bookkeeper</pattern> - <shadedPattern>hidden.bkjournal.org.apache.bookkeeper</shadedPattern> - </relocation> - <relocation> - <pattern>org.apache.zookeeper</pattern> - <shadedPattern>hidden.bkjournal.org.apache.zookeeper</shadedPattern> - </relocation> - <relocation> - <pattern>org.jboss.netty</pattern> - <shadedPattern>hidden.bkjournal.org.jboss.netty</shadedPattern> - </relocation> - </relocations> + <artifactItems> + <artifactItem> + <groupId>org.apache.bookkeeper</groupId> + <artifactId>bookkeeper-server</artifactId> + <type>jar</type> + </artifactItem> + </artifactItems> + <outputDirectory>${project.build.directory}/lib</outputDirectory> </configuration> </execution> </executions> Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Tue Aug 19 23:49:39 2014 @@ -237,7 +237,7 @@ public class BookKeeperJournalManager im zkPathLatch.countDown(); } }; - ZkUtils.createFullPathOptimistic(zkc, zkAvailablePath, new byte[0], + ZkUtils.asyncCreateFullPathOptimistic(zkc, zkAvailablePath, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, callback, null); try { Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java Tue Aug 19 23:49:39 2014 @@ -149,13 +149,16 @@ class BKJMUtil { int checkBookiesUp(int count, int timeout) throws Exception { ZooKeeper zkc = connectZooKeeper(); try { - boolean up = false; int mostRecentSize = 0; for (int i = 0; i < timeout; i++) { try { List<String> children = zkc.getChildren("/ledgers/available", false); mostRecentSize = children.size(); + // Skip 'readonly znode' which is used for keeping R-O bookie details + if (children.contains("readonly")) { + mostRecentSize = children.size() - 1; + } if (LOG.isDebugEnabled()) { LOG.debug("Found " + mostRecentSize + " bookies up, " + "waiting for " + count); @@ -166,7 +169,6 @@ class BKJMUtil { } } if (mostRecentSize == count) { - up = true; break; } } catch (KeeperException e) { Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java Tue Aug 19 23:49:39 2014 @@ -23,13 +23,10 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.ha.TestStandbyCheckpoints; -import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import com.google.common.collect.ImmutableList; - /** * Runs the same tests as TestStandbyCheckpoints, but * using a bookkeeper journal manager as the shared directory @@ -43,19 +40,11 @@ public class TestBookKeeperHACheckpoints @Override @Before public void setupCluster() throws Exception { - Configuration conf = new Configuration(); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 5); - conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + Configuration conf = setupCommonConfig(); conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil.createJournalURI("/checkpointing" + journalCount++) .toString()); BKJMUtil.addJournalManagerDefinition(conf); - conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true); - conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, SlowCodec.class - .getCanonicalName()); - CompressionCodecFactory.setCodecClasses(conf, ImmutableList - .<Class> of(SlowCodec.class)); MiniDFSNNTopology topology = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ns1") .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)) Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh Tue Aug 19 23:49:39 2014 @@ -57,9 +57,9 @@ excludeFilenameRemote=$("$HADOOP_PREFIX/ if [ "$excludeFilenameRemote" = '' ] ; then echo \ - "Error: hdfs getconf -excludeFile returned empty string, " \ - "please setup dfs.hosts.exclude in hdfs-site.xml in local cluster " \ - "configuration and on all namenodes" + "Error: hdfs getconf -excludeFile returned empty string, " \ + "please setup dfs.hosts.exclude in hdfs-site.xml in local cluster " \ + "configuration and on all namenodes" exit 1 fi Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs Tue Aug 19 23:49:39 2014 @@ -15,194 +15,237 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Environment Variables -# -# JSVC_HOME home directory of jsvc binary. Required for starting secure -# datanode. -# -# JSVC_OUTFILE path to jsvc output file. Defaults to -# $HADOOP_LOG_DIR/jsvc.out. -# -# JSVC_ERRFILE path to jsvc error file. Defaults to $HADOOP_LOG_DIR/jsvc.err. - -bin=`which $0` -bin=`dirname ${bin}` -bin=`cd "$bin" > /dev/null; pwd` - -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh - -function print_usage(){ - echo "Usage: hdfs [--config confdir] COMMAND" +function hadoop_usage +{ + echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)] COMMAND" echo " where COMMAND is one of:" - echo " dfs run a filesystem command on the file systems supported in Hadoop." - echo " namenode -format format the DFS filesystem" - echo " secondarynamenode run the DFS secondary namenode" - echo " namenode run the DFS namenode" - echo " journalnode run the DFS journalnode" - echo " zkfc run the ZK Failover Controller daemon" + echo " balancer run a cluster balancing utility" + echo " cacheadmin configure the HDFS cache" + echo " classpath prints the class path needed to get the" + echo " Hadoop jar and the required libraries" echo " datanode run a DFS datanode" + echo " dfs run a filesystem command on the file system" echo " dfsadmin run a DFS admin client" - echo " haadmin run a DFS HA admin client" - echo " fsck run a DFS filesystem checking utility" - echo " balancer run a cluster balancing utility" - echo " jmxget get JMX exported values from NameNode or DataNode." - echo " oiv apply the offline fsimage viewer to an fsimage" - echo " oev apply the offline edits viewer to an edits file" echo " fetchdt fetch a delegation token from the NameNode" + echo " fsck run a DFS filesystem checking utility" echo " getconf get config values from configuration" echo " groups get the groups which users belong to" - echo " snapshotDiff diff two snapshots of a directory or diff the" - echo " current directory contents with a snapshot" + echo " haadmin run a DFS HA admin client" + echo " jmxget get JMX exported values from NameNode or DataNode." + echo " journalnode run the DFS journalnode" echo " lsSnapshottableDir list all snapshottable dirs owned by the current user" - echo " Use -help to see options" - echo " portmap run a portmap service" + echo " Use -help to see options" + echo " namenode run the DFS namenode" + echo " Use -format to initialize the DFS filesystem" echo " nfs3 run an NFS version 3 gateway" - echo " cacheadmin configure the HDFS cache" + echo " oev apply the offline edits viewer to an edits file" + echo " oiv apply the offline fsimage viewer to an fsimage" + echo " oiv_legacy apply the offline fsimage viewer to a legacy fsimage" + echo " portmap run a portmap service" + echo " secondarynamenode run the DFS secondary namenode" + echo " snapshotDiff diff two snapshots of a directory or diff the" + echo " current directory contents with a snapshot" + echo " zkfc run the ZK Failover Controller daemon" echo "" echo "Most commands print help when invoked w/o parameters." } -if [ $# = 0 ]; then - print_usage - exit +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi + +if [[ $# = 0 ]]; then + hadoop_exit_with_usage 1 fi COMMAND=$1 shift -case $COMMAND in - # usage flags - --help|-help|-h) - print_usage +case ${COMMAND} in + balancer) + CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}" + ;; + cacheadmin) + CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin + ;; + classpath) + hadoop_finalize + echo "${CLASSPATH}" exit - ;; -esac - -# Determine if we're starting a secure datanode, and if so, redefine appropriate variables -if [ "$COMMAND" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then - if [ -n "$JSVC_HOME" ]; then - if [ -n "$HADOOP_SECURE_DN_PID_DIR" ]; then - HADOOP_PID_DIR=$HADOOP_SECURE_DN_PID_DIR + ;; + datanode) + daemon="true" + # Determine if we're starting a secure datanode, and + # if so, redefine appropriate variables + if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then + secure_service="true" + secure_user="${HADOOP_SECURE_DN_USER}" + + # backward compatiblity + HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}" + HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}" + + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS} ${HADOOP_DATANODE_OPTS}" + CLASS="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter" + else + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}" + CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode' fi - - if [ -n "$HADOOP_SECURE_DN_LOG_DIR" ]; then - HADOOP_LOG_DIR=$HADOOP_SECURE_DN_LOG_DIR - HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" + ;; + dfs) + CLASS=org.apache.hadoop.fs.FsShell + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + dfsadmin) + CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + fetchdt) + CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher + ;; + fsck) + CLASS=org.apache.hadoop.hdfs.tools.DFSck + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + getconf) + CLASS=org.apache.hadoop.hdfs.tools.GetConf + ;; + groups) + CLASS=org.apache.hadoop.hdfs.tools.GetGroups + ;; + haadmin) + CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin + CLASSPATH="${CLASSPATH}:${TOOL_PATH}" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + journalnode) + daemon="true" + CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode' + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}" + ;; + jmxget) + CLASS=org.apache.hadoop.hdfs.tools.JMXGet + ;; + lsSnapshottableDir) + CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir + ;; + namenode) + daemon="true" + CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}" + ;; + nfs3) + daemon="true" + if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then + secure_service="true" + secure_user="${HADOOP_PRIVILEGED_NFS_USER}" + + # backward compatiblity + HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}" + HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}" + + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS} ${HADOOP_NFS3_OPTS}" + CLASS=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter + else + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}" + CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 fi - - HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER - HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING" - starting_secure_dn="true" - else - echo "It looks like you're trying to start a secure DN, but \$JSVC_HOME"\ - "isn't set. Falling back to starting insecure DN." - fi -fi + ;; + oev) + CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer + ;; + oiv) + CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB + ;; + oiv_legacy) + CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer + ;; + portmap) + daemon="true" + CLASS=org.apache.hadoop.portmap.Portmap + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}" + ;; + secondarynamenode) + daemon="true" + CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode' + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}" + ;; + snapshotDiff) + CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff + ;; + zkfc) + daemon="true" + CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController' + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}" + ;; + -*) + hadoop_exit_with_usage 1 + ;; + *) + CLASS="${COMMAND}" + ;; +esac -if [ "$COMMAND" = "namenode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS" -elif [ "$COMMAND" = "zkfc" ] ; then - CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_ZKFC_OPTS" -elif [ "$COMMAND" = "secondarynamenode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS" -elif [ "$COMMAND" = "datanode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode' - if [ "$starting_secure_dn" = "true" ]; then - HADOOP_OPTS="$HADOOP_OPTS -jvm server $HADOOP_DATANODE_OPTS" - else - HADOOP_OPTS="$HADOOP_OPTS -server $HADOOP_DATANODE_OPTS" +if [[ -n "${secure_service}" ]]; then + HADOOP_SECURE_USER="${secure_user}" + if hadoop_verify_secure_prereq; then + hadoop_setup_secure_service + priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.out" + priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.err" + priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.pid" + daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" + daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid" fi -elif [ "$COMMAND" = "journalnode" ] ; then - CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOURNALNODE_OPTS" -elif [ "$COMMAND" = "dfs" ] ; then - CLASS=org.apache.hadoop.fs.FsShell - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "dfsadmin" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "haadmin" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin - CLASSPATH=${CLASSPATH}:${TOOL_PATH} - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "fsck" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DFSck - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "balancer" ] ; then - CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS" -elif [ "$COMMAND" = "jmxget" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.JMXGet -elif [ "$COMMAND" = "oiv" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB -elif [ "$COMMAND" = "oev" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer -elif [ "$COMMAND" = "fetchdt" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher -elif [ "$COMMAND" = "getconf" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.GetConf -elif [ "$COMMAND" = "groups" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.GetGroups -elif [ "$COMMAND" = "snapshotDiff" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff -elif [ "$COMMAND" = "lsSnapshottableDir" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir -elif [ "$COMMAND" = "portmap" ] ; then - CLASS=org.apache.hadoop.portmap.Portmap - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_PORTMAP_OPTS" -elif [ "$COMMAND" = "nfs3" ] ; then - CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NFS3_OPTS" -elif [ "$COMMAND" = "cacheadmin" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin else - CLASS="$COMMAND" + daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" + daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid" fi -export CLASSPATH=$CLASSPATH - -HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}" - -# Check to see if we should start a secure datanode -if [ "$starting_secure_dn" = "true" ]; then - if [ "$HADOOP_PID_DIR" = "" ]; then - HADOOP_SECURE_DN_PID="/tmp/hadoop_secure_dn.pid" +if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then + # shellcheck disable=SC2034 + HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" + if [[ -n "${secure_service}" ]]; then + # shellcheck disable=SC2034 + HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log" else - HADOOP_SECURE_DN_PID="$HADOOP_PID_DIR/hadoop_secure_dn.pid" + # shellcheck disable=SC2034 + HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log" fi +fi - JSVC=$JSVC_HOME/jsvc - if [ ! -f $JSVC ]; then - echo "JSVC_HOME is not set correctly so jsvc cannot be found. Jsvc is required to run secure datanodes. " - echo "Please download and install jsvc from http://archive.apache.org/dist/commons/daemon/binaries/ "\ - "and set JSVC_HOME to the directory containing the jsvc binary." - exit - fi +hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}" +hadoop_finalize - if [[ ! $JSVC_OUTFILE ]]; then - JSVC_OUTFILE="$HADOOP_LOG_DIR/jsvc.out" - fi +export CLASSPATH - if [[ ! $JSVC_ERRFILE ]]; then - JSVC_ERRFILE="$HADOOP_LOG_DIR/jsvc.err" +if [[ -n "${daemon}" ]]; then + if [[ -n "${secure_service}" ]]; then + hadoop_secure_daemon_handler \ + "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\ + "${daemon_pidfile}" "${daemon_outfile}" \ + "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@" + else + hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\ + "${daemon_pidfile}" "${daemon_outfile}" "$@" fi - - exec "$JSVC" \ - -Dproc_$COMMAND -outfile "$JSVC_OUTFILE" \ - -errfile "$JSVC_ERRFILE" \ - -pidfile "$HADOOP_SECURE_DN_PID" \ - -nodetach \ - -user "$HADOOP_SECURE_DN_USER" \ - -cp "$CLASSPATH" \ - $JAVA_HEAP_MAX $HADOOP_OPTS \ - org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter "$@" + exit $? else - # run it - exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@" + # shellcheck disable=SC2086 + hadoop_java_exec "${COMMAND}" "${CLASS}" "$@" fi - Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh Tue Aug 19 23:49:39 2014 @@ -18,19 +18,67 @@ # included in all the hdfs scripts with source command # should not be executed directly -bin=`which "$0"` -bin=`dirname "${bin}"` -bin=`cd "$bin"; pwd` +function hadoop_subproject_init +{ + if [ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]; then + . "${HADOOP_CONF_DIR}/hdfs-env.sh" + fi + + # at some point in time, someone thought it would be a good idea to + # create separate vars for every subproject. *sigh* + # let's perform some overrides and setup some defaults for bw compat + # this way the common hadoop var's == subproject vars and can be + # used interchangeable from here on out + # ... + # this should get deprecated at some point. + HADOOP_LOG_DIR="${HADOOP_HDFS_LOG_DIR:-$HADOOP_LOG_DIR}" + HADOOP_HDFS_LOG_DIR="${HADOOP_LOG_DIR}" + + HADOOP_LOGFILE="${HADOOP_HDFS_LOGFILE:-$HADOOP_LOGFILE}" + HADOOP_HDFS_LOGFILE="${HADOOP_LOGFILE}" + + HADOOP_NICENESS=${HADOOP_HDFS_NICENESS:-$HADOOP_NICENESS} + HADOOP_HDFS_NICENESS="${HADOOP_NICENESS}" + + HADOOP_STOP_TIMEOUT=${HADOOP_HDFS_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT} + HADOOP_HDFS_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}" + + HADOOP_PID_DIR="${HADOOP_HDFS_PID_DIR:-$HADOOP_PID_DIR}" + HADOOP_HDFS_PID_DIR="${HADOOP_PID_DIR}" + + HADOOP_ROOT_LOGGER=${HADOOP_HDFS_ROOT_LOGGER:-$HADOOP_ROOT_LOGGER} + HADOOP_HDFS_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}" + + HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME_DIR}" + + HADOOP_IDENT_STRING="${HADOOP_HDFS_IDENT_STRING:-$HADOOP_IDENT_STRING}" + HADOOP_HDFS_IDENT_STRING="${HADOOP_IDENT_STRING}" + + # turn on the defaults + + export HADOOP_NAMENODE_OPTS=${HADOOP_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"} + export HADOOP_SECONDARYNAMENODE_OPTS=${HADOOP_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"} + export HADOOP_DATANODE_OPTS=${HADOOP_DATANODE_OPTS:-"-Dhadoop.security.logger=ERROR,RFAS"} + export HADOOP_DN_SECURE_EXTRA_OPTS=${HADOOP_DN_SECURE_EXTRA_OPTS:-"-jvm server"} + export HADOOP_NFS3_SECURE_EXTRA_OPTS=${HADOOP_NFS3_SECURE_EXTRA_OPTS:-"-jvm server"} + export HADOOP_PORTMAP_OPTS=${HADOOP_PORTMAP_OPTS:-"-Xmx512m"} + + +} + +if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then + _hd_this="${BASH_SOURCE-$0}" + HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hd_this}")" >/dev/null && pwd -P) +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then - . ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh + . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then - . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh + . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then - . "$HADOOP_HOME"/libexec/hadoop-config.sh + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else - echo "Hadoop common not found." - exit + echo "ERROR: Hadoop common not found." 2>&1 + exit 1 fi + Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd Tue Aug 19 23:49:39 2014 @@ -47,7 +47,7 @@ if "%1" == "--config" ( goto print_usage ) - set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir + set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin for %%i in ( %hdfscommands% ) do ( if %hdfs-command% == %%i set hdfscommand=true ) @@ -146,6 +146,10 @@ goto :eof set CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir goto :eof +:cacheadmin + set CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin + goto :eof + @rem This changes %1, %2 etc. Hence those cannot be used after calling this. :make_command_arguments if "%1" == "--config" ( @@ -193,6 +197,7 @@ goto :eof @echo current directory contents with a snapshot @echo lsSnapshottableDir list all snapshottable dirs owned by the current user @echo Use -help to see options + @echo cacheadmin configure the HDFS cache @echo. @echo Most commands print help when invoked w/o parameters. Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh Tue Aug 19 23:49:39 2014 @@ -20,24 +20,40 @@ # This script refreshes all namenodes, it's a simple wrapper # for dfsadmin to support multiple namenodes. -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi -namenodes=$("$HADOOP_PREFIX/bin/hdfs" getconf -nnRpcAddresses) -if [ "$?" != '0' ] ; then errorFlag='1' ; +namenodes=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -nnRpcAddresses) +if [[ "$?" != '0' ]] ; then + errorFlag='1' ; else - for namenode in $namenodes ; do - echo "Refreshing namenode [$namenode]" - "$HADOOP_PREFIX/bin/hdfs" dfsadmin -fs hdfs://$namenode -refreshNodes - if [ "$?" != '0' ] ; then errorFlag='1' ; fi + for namenode in ${namenodes} ; do + echo "Refreshing namenode [${namenode}]" + "${HADOOP_HDFS_HOME}/bin/hdfs" dfsadmin \ + -fs hdfs://${namenode} -refreshNodes + if [[ "$?" != '0' ]]; then + errorFlag='1' + fi done fi -if [ "$errorFlag" = '1' ] ; then +if [[ "${errorFlag}" = '1' ]] ; then echo "Error: refresh of namenodes failed, see error messages above." exit 1 else Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh Tue Aug 19 23:49:39 2014 @@ -15,13 +15,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function usage +{ + echo "Usage: start-balancer.sh [--config confdir] [-policy <policy>] [-threshold <threshold>]" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi # Start balancer daemon. -"$HADOOP_PREFIX"/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start balancer $@ +exec "${bin}/hadoop-daemon.sh" --config "${HADOOP_CONF_DIR}" start balancer "$@" Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh Tue Aug 19 23:49:39 2014 @@ -20,98 +20,128 @@ # Optinally upgrade or rollback dfs state. # Run this on master node. -usage="Usage: start-dfs.sh [-upgrade|-rollback] [other options such as -clusterId]" +function hadoop_usage +{ + echo "Usage: start-dfs.sh [-upgrade|-rollback] [-clusterId]" +} + +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh # get arguments -if [ $# -ge 1 ]; then - nameStartOpt="$1" - shift - case "$nameStartOpt" in - (-upgrade) - ;; - (-rollback) - dataStartOpt="$nameStartOpt" - ;; - (*) - echo $usage - exit 1 - ;; - esac +if [[ $# -ge 1 ]]; then + nameStartOpt="$1" + shift + case "$nameStartOpt" in + -upgrade) + ;; + -rollback) + dataStartOpt="$nameStartOpt" + ;; + *) + hadoop_exit_with_usage 1 + ;; + esac fi + #Add other possible options nameStartOpt="$nameStartOpt $@" #--------------------------------------------------------- # namenodes -NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes) +NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes 2>/dev/null) + +if [[ -z "${NAMENODES}" ]]; then + NAMENODES=$(hostname) +fi echo "Starting namenodes on [$NAMENODES]" -"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$NAMENODES" \ - --script "$bin/hdfs" start namenode $nameStartOpt +"${bin}/hadoop-daemons.sh" \ +--config "${HADOOP_CONF_DIR}" \ +--hostnames "${NAMENODES}" \ +start namenode ${nameStartOpt} #--------------------------------------------------------- # datanodes (using default slaves file) -if [ -n "$HADOOP_SECURE_DN_USER" ]; then - echo \ - "Attempting to start secure cluster, skipping datanodes. " \ - "Run start-secure-dns.sh as root to complete startup." +if [[ -n "${HADOOP_SECURE_DN_USER}" ]] && +[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then + echo "ERROR: Attempting to start secure cluster, skipping datanodes. " + echo "Run start-secure-dns.sh as root or configure " + echo "\${HADOOP_SECURE_COMMAND} to complete startup." else - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --script "$bin/hdfs" start datanode $dataStartOpt + + echo "Starting datanodes" + + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + start datanode ${dataStartOpt} fi #--------------------------------------------------------- # secondary namenodes (if any) -SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes 2>/dev/null) +SECONDARY_NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -secondarynamenodes 2>/dev/null) -if [ -n "$SECONDARY_NAMENODES" ]; then - echo "Starting secondary namenodes [$SECONDARY_NAMENODES]" +if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then + SECONDARY_NAMENODES=$(hostname) +fi - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$SECONDARY_NAMENODES" \ - --script "$bin/hdfs" start secondarynamenode +if [[ -n "${SECONDARY_NAMENODES}" ]]; then + echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]" + + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${SECONDARY_NAMENODES}" \ + start secondarynamenode fi #--------------------------------------------------------- # quorumjournal nodes (if any) -SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-) +SHARED_EDITS_DIR=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.namenode.shared.edits.dir 2>&-) -case "$SHARED_EDITS_DIR" in -qjournal://*) - JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') - echo "Starting journal nodes [$JOURNAL_NODES]" - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$JOURNAL_NODES" \ - --script "$bin/hdfs" start journalnode ;; +case "${SHARED_EDITS_DIR}" in + qjournal://*) + JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') + echo "Starting journal nodes [${JOURNAL_NODES}]" + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${JOURNAL_NODES}" \ + start journalnode + ;; esac #--------------------------------------------------------- # ZK Failover controllers, if auto-HA is enabled -AUTOHA_ENABLED=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.ha.automatic-failover.enabled) -if [ "$(echo "$AUTOHA_ENABLED" | tr A-Z a-z)" = "true" ]; then - echo "Starting ZK Failover Controllers on NN hosts [$NAMENODES]" - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$NAMENODES" \ - --script "$bin/hdfs" start zkfc +AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]') +if [[ "${AUTOHA_ENABLED}" = "true" ]]; then + echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]" + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${NAMENODES}" \ + start zkfc fi # eof Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh Tue Aug 19 23:49:39 2014 @@ -17,17 +17,33 @@ # Run as root to start secure datanodes in a security-enabled cluster. -usage="Usage (run as root in order to start secure datanodes): start-secure-dns.sh" -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function hadoop_usage { + echo "Usage: start-secure-dns.sh" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then - "$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else - echo $usage + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi + +if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then + exec "${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" start datanode "${dataStartOpt}" +else + echo hadoop_usage_and_exit 1 fi Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh Tue Aug 19 23:49:39 2014 @@ -15,14 +15,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function hadoop_usage +{ + echo "Usage: stop-balancer.sh [--config confdir]" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi # Stop balancer daemon. # Run this on the machine where the balancer is running -"$HADOOP_PREFIX"/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop balancer +"${bin}/hadoop-daemon.sh" --config "${HADOOP_CONF_DIR}" stop balancer Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh Tue Aug 19 23:49:39 2014 @@ -15,75 +15,100 @@ # See the License for the specific language governing permissions and # limitations under the License. -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function hadoop_usage +{ + echo "Usage: start-balancer.sh [--config confdir] [-policy <policy>] [-threshold <threshold>]" +} + +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi #--------------------------------------------------------- # namenodes -NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes) +NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes) echo "Stopping namenodes on [$NAMENODES]" -"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$NAMENODES" \ - --script "$bin/hdfs" stop namenode +"${bin}/hadoop-daemons.sh" \ +--config "${HADOOP_CONF_DIR}" \ +--hostnames "${NAMENODES}" \ +stop namenode #--------------------------------------------------------- # datanodes (using default slaves file) -if [ -n "$HADOOP_SECURE_DN_USER" ]; then +if [[ -n "${HADOOP_SECURE_DN_USER}" ]] && +[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then echo \ - "Attempting to stop secure cluster, skipping datanodes. " \ - "Run stop-secure-dns.sh as root to complete shutdown." + "ERROR: Attempting to stop secure cluster, skipping datanodes. " \ + "Run stop-secure-dns.sh as root to complete shutdown." else - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --script "$bin/hdfs" stop datanode + + echo "Stopping datanodes" + + "${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" stop datanode fi #--------------------------------------------------------- # secondary namenodes (if any) -SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes 2>/dev/null) +SECONDARY_NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -secondarynamenodes 2>/dev/null) -if [ -n "$SECONDARY_NAMENODES" ]; then - echo "Stopping secondary namenodes [$SECONDARY_NAMENODES]" +if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then + SECONDARY_NAMENODES=$(hostname) +fi - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$SECONDARY_NAMENODES" \ - --script "$bin/hdfs" stop secondarynamenode +if [[ -n "${SECONDARY_NAMENODES}" ]]; then + echo "Stopping secondary namenodes [${SECONDARY_NAMENODES}]" + + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${SECONDARY_NAMENODES}" \ + stop secondarynamenode fi #--------------------------------------------------------- # quorumjournal nodes (if any) -SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-) +SHARED_EDITS_DIR=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.namenode.shared.edits.dir 2>&-) -case "$SHARED_EDITS_DIR" in -qjournal://*) - JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') - echo "Stopping journal nodes [$JOURNAL_NODES]" - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$JOURNAL_NODES" \ - --script "$bin/hdfs" stop journalnode ;; +case "${SHARED_EDITS_DIR}" in + qjournal://*) + JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') + echo "Stopping journal nodes [${JOURNAL_NODES}]" + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${JOURNAL_NODES}" \ + stop journalnode + ;; esac #--------------------------------------------------------- # ZK Failover controllers, if auto-HA is enabled -AUTOHA_ENABLED=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.ha.automatic-failover.enabled) -if [ "$(echo "$AUTOHA_ENABLED" | tr A-Z a-z)" = "true" ]; then - echo "Stopping ZK Failover Controllers on NN hosts [$NAMENODES]" - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$NAMENODES" \ - --script "$bin/hdfs" stop zkfc +AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]') +if [[ "${AUTOHA_ENABLED}" = "true" ]]; then + echo "Stopping ZK Failover Controllers on NN hosts [${NAMENODES}]" + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${NAMENODES}" \ + stop zkfc fi # eof Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh Tue Aug 19 23:49:39 2014 @@ -17,17 +17,33 @@ # Run as root to start secure datanodes in a security-enabled cluster. -usage="Usage (run as root in order to stop secure datanodes): stop-secure-dns.sh" -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function hadoop_usage { + echo "Usage (run as root in order to stop secure datanodes): stop-secure-dns.sh" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then - "$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else - echo $usage + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi + +if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then + "${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" stop datanode +else + hadoop_exit_with_usage 1 fi Propchange: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1606534 Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1588388-1619000 Merged /hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1588992-1596568 Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java Tue Aug 19 23:49:39 2014 @@ -25,6 +25,7 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import org.apache.hadoop.classification.InterfaceAudience; @@ -32,6 +33,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.hdfs.CorruptFileBlockIterator; @@ -67,9 +69,8 @@ public class Hdfs extends AbstractFileSy * This constructor has the signature needed by * {@link AbstractFileSystem#createFileSystem(URI, Configuration)} * - * @param theUri - * which must be that of Hdfs - * @param conf + * @param theUri which must be that of Hdfs + * @param conf configuration * @throws IOException */ Hdfs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException { @@ -116,7 +117,7 @@ public class Hdfs extends AbstractFileSy @Override public FileChecksum getFileChecksum(Path f) throws IOException, UnresolvedLinkException { - return dfs.getFileChecksum(getUriPath(f)); + return dfs.getFileChecksum(getUriPath(f), Long.MAX_VALUE); } @Override @@ -415,6 +416,43 @@ public class Hdfs extends AbstractFileSy public AclStatus getAclStatus(Path path) throws IOException { return dfs.getAclStatus(getUriPath(path)); } + + @Override + public void setXAttr(Path path, String name, byte[] value, + EnumSet<XAttrSetFlag> flag) throws IOException { + dfs.setXAttr(getUriPath(path), name, value, flag); + } + + @Override + public byte[] getXAttr(Path path, String name) throws IOException { + return dfs.getXAttr(getUriPath(path), name); + } + + @Override + public Map<String, byte[]> getXAttrs(Path path) throws IOException { + return dfs.getXAttrs(getUriPath(path)); + } + + @Override + public Map<String, byte[]> getXAttrs(Path path, List<String> names) + throws IOException { + return dfs.getXAttrs(getUriPath(path), names); + } + + @Override + public List<String> listXAttrs(Path path) throws IOException { + return dfs.listXAttrs(getUriPath(path)); + } + + @Override + public void removeXAttr(Path path, String name) throws IOException { + dfs.removeXAttr(getUriPath(path), name); + } + + @Override + public void access(Path path, final FsAction mode) throws IOException { + dfs.checkAccess(getUriPath(path), mode); + } /** * Renew an existing delegation token. Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java Tue Aug 19 23:49:39 2014 @@ -51,6 +51,7 @@ public interface BlockReader extends Byt * Returns an estimate of the number of bytes that can be read * (or skipped over) from this input stream without performing * network I/O. + * This may return more than what is actually present in the block. */ int available() throws IOException; Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java Tue Aug 19 23:49:39 2014 @@ -744,7 +744,8 @@ public class BlockReaderFactory implemen } } try { - Peer peer = remotePeerFactory.newConnectedPeer(inetSocketAddress); + Peer peer = remotePeerFactory.newConnectedPeer(inetSocketAddress, token, + datanode); if (LOG.isTraceEnabled()) { LOG.trace("nextTcpPeer: created newConnectedPeer " + peer); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java Tue Aug 19 23:49:39 2014 @@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.dat import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.shortcircuit.ClientMmap; import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplica; -import org.apache.hadoop.hdfs.util.DirectBufferPool; +import org.apache.hadoop.util.DirectBufferPool; import org.apache.hadoop.util.DataChecksum; import com.google.common.annotations.VisibleForTesting; @@ -182,7 +182,7 @@ class BlockReaderLocal implements BlockR /** * Maximum amount of readahead we'll do. This will always be at least the, - * size of a single chunk, even if {@link zeroReadaheadRequested} is true. + * size of a single chunk, even if {@link #zeroReadaheadRequested} is true. * The reason is because we need to do a certain amount of buffering in order * to do checksumming. * Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java Tue Aug 19 23:49:39 2014 @@ -40,7 +40,7 @@ import org.apache.hadoop.hdfs.protocol.E import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; import org.apache.hadoop.hdfs.shortcircuit.ClientMmap; -import org.apache.hadoop.hdfs.util.DirectBufferPool; +import org.apache.hadoop.util.DirectBufferPool; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.UserGroupInformation; @@ -187,7 +187,7 @@ class BlockReaderLocalLegacy implements userGroupInformation = UserGroupInformation.getCurrentUser(); } pathinfo = getBlockPathInfo(userGroupInformation, blk, node, - configuration, conf.hdfsTimeout, token, + configuration, conf.socketTimeout, token, conf.connectToDnViaHostname); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java Tue Aug 19 23:49:39 2014 @@ -191,7 +191,8 @@ class BlockStorageLocationUtil { /** * Group the per-replica {@link VolumeId} info returned from - * {@link DFSClient#queryDatanodesForHdfsBlocksMetadata(Map)} to be associated + * {@link DFSClient#queryDatanodesForHdfsBlocksMetadata(Map)} to be + * associated * with the corresponding {@link LocatedBlock}. * * @param blocks
