Author: atm
Date: Sat Apr 13 02:13:59 2013
New Revision: 1467538
URL: http://svn.apache.org/r1467538
Log:
HDFS-347. DFS read performance suboptimal when client co-located on nodes with
data. Contributed by Colin Patrick McCabe.
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/
- copied from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
- copied unchanged from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/exception.c
- copied unchanged from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/native/src/exception.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/exception.h
- copied unchanged from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/native/src/exception.h
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/
- copied from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/
- copied from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
- copied unchanged from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/
- copied from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TemporarySocketDirectory.java
- copied unchanged from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TemporarySocketDirectory.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
- copied unchanged from r1467533,
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(props changed)
hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
Propchange: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/CHANGES.txt:r1430995-1467533
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml?rev=1467538&r1=1467537&r2=1467538&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml Sat Apr 13
02:13:59 2013
@@ -522,6 +522,7 @@
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
+
<javahClassName>org.apache.hadoop.net.unix.DomainSocket</javahClassName>
</javahClassNames>
<javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
</configuration>
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1467538&r1=1467537&r2=1467538&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
Sat Apr 13 02:13:59 2013
@@ -163,10 +163,10 @@ add_executable(test_bulk_crc32
${D}/util/bulk_crc32.c
${T}/util/test_bulk_crc32.c
)
-set_property(SOURCE main.cpp PROPERTY INCLUDE_DIRECTORIES "\"-Werror\"
\"-Wall\"")
SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
add_dual_library(hadoop
+ main/native/src/exception.c
${D}/io/compress/lz4/Lz4Compressor.c
${D}/io/compress/lz4/Lz4Decompressor.c
${D}/io/compress/lz4/lz4.c
@@ -177,6 +177,7 @@ add_dual_library(hadoop
${D}/io/nativeio/NativeIO.c
${D}/io/nativeio/errno_enum.c
${D}/io/nativeio/file_descriptor.c
+ ${D}/net/unix/DomainSocket.c
${D}/security/JniBasedUnixGroupsMapping.c
${D}/security/JniBasedUnixGroupsNetgroupMapping.c
${D}/security/getGroup.c
Propchange:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/docs:r1430995-1467533
Propchange:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java:r1430995-1467533
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1467538&r1=1467537&r2=1467538&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
Sat Apr 13 02:13:59 2013
@@ -743,6 +743,21 @@ public class Configuration implements It
return value.trim();
}
}
+
+ /**
+ * Get the value of the <code>name</code> property as a trimmed
<code>String</code>,
+ * <code>defaultValue</code> if no such property exists.
+ * See @{Configuration#getTrimmed} for more details.
+ *
+ * @param name the property name.
+ * @param defaultValue the property default value.
+ * @return the value of the <code>name</code> or defaultValue
+ * if it is not set.
+ */
+ public String getTrimmed(String name, String defaultValue) {
+ String ret = getTrimmed(name);
+ return ret == null ? defaultValue : ret;
+ }
/**
* Get the value of the <code>name</code> property, without doing
@@ -877,7 +892,7 @@ public class Configuration implements It
}
return result;
}
-
+
/**
* Get the value of the <code>name</code> property as an <code>int</code>.
*
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java?rev=1467538&r1=1467537&r2=1467538&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
Sat Apr 13 02:13:59 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.net;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import java.io.InputStream;
import java.net.Socket;
import java.net.SocketTimeoutException;
@@ -37,7 +38,8 @@ import java.nio.channels.SelectionKey;
* IllegalBlockingModeException.
* Please use {@link SocketOutputStream} for writing.
*/
-class SocketInputStream extends InputStream
[email protected]("HDFS")
+public class SocketInputStream extends InputStream
implements ReadableByteChannel {
private Reader reader;
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java?rev=1467538&r1=1467537&r2=1467538&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
Sat Apr 13 02:13:59 2013
@@ -260,4 +260,8 @@ public class SocketOutputStream extends
throws IOException {
transferToFully(fileCh, position, count, null, null);
}
+
+ public void setTimeout(int timeoutMs) {
+ writer.setTimeout(timeoutMs);
+ }
}
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java?rev=1467538&r1=1467537&r2=1467538&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
Sat Apr 13 02:13:59 2013
@@ -104,7 +104,7 @@ public class DataChecksum implements Che
( (bytes[offset+2] & 0xff) << 16 ) |
( (bytes[offset+3] & 0xff) << 8 ) |
( (bytes[offset+4] & 0xff) );
- return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
+ return newDataChecksum( Type.valueOf(bytes[offset]), bytesPerChecksum );
}
/**
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h?rev=1467538&r1=1467537&r2=1467538&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
Sat Apr 13 02:13:59 2013
@@ -180,6 +180,10 @@ static FARPROC WINAPI do_dlsym(JNIEnv *e
THROW(env, "java/lang/InternalError", exception_msg); \
}
+#define RETRY_ON_EINTR(ret, expr) do { \
+ ret = expr; \
+} while ((ret == -1) && (errno == EINTR));
+
#endif
//vim: sw=2: ts=2: et
Propchange:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/core:r1430995-1467533
