Author: szetszwo
Date: Sat Nov 9 01:34:22 2013
New Revision: 1540239
URL: http://svn.apache.org/r1540239
Log:
merge of r1535792 through r1540238 from trunk.
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1539897-1540238
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1540239&r1=1540238&r2=1540239&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
Sat Nov 9 01:34:22 2013
@@ -23,7 +23,9 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
+import java.lang.reflect.Field;
import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -34,10 +36,11 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import sun.misc.Unsafe;
+
import com.google.common.annotations.VisibleForTesting;
/**
@@ -271,6 +274,26 @@ public class NativeIO {
}
munlock_native(buffer, len);
}
+
+ /**
+ * Unmaps the block from memory. See munmap(2).
+ *
+ * There isn't any portable way to unmap a memory region in Java.
+ * So we use the sun.nio method here.
+ * Note that unmapping a memory region could cause crashes if code
+ * continues to reference the unmapped code. However, if we don't
+ * manually unmap the memory, we are dependent on the finalizer to
+ * do it, and we have no idea when the finalizer will run.
+ *
+ * @param buffer The buffer to unmap.
+ */
+ public static void munmap(MappedByteBuffer buffer) {
+ if (buffer instanceof sun.nio.ch.DirectBuffer) {
+ sun.misc.Cleaner cleaner =
+ ((sun.nio.ch.DirectBuffer)buffer).cleaner();
+ cleaner.clean();
+ }
+ }
/** Linux only methods used for getOwner() implementation */
private static native long getUIDforFDOwnerforOwner(FileDescriptor fd)
throws IOException;
@@ -539,6 +562,21 @@ public class NativeIO {
private static native long getMemlockLimit0();
+ /**
+ * @return the operating system's page size.
+ */
+ public static long getOperatingSystemPageSize() {
+ try {
+ Field f = Unsafe.class.getDeclaredField("theUnsafe");
+ f.setAccessible(true);
+ Unsafe unsafe = (Unsafe)f.get(null);
+ return unsafe.pageSize();
+ } catch (Throwable e) {
+ LOG.warn("Unable to get operating system page size. Guessing 4096.", e);
+ return 4096;
+ }
+ }
+
private static class CachedUid {
final long timestamp;
final String username;
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java?rev=1540239&r1=1540238&r2=1540239&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java
Sat Nov 9 01:34:22 2013
@@ -18,9 +18,9 @@
package org.apache.hadoop.io.retry;
import java.lang.reflect.Method;
-import java.net.UnknownHostException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.RetriableException;
/**
* A dummy invocation handler extending RetryInvocationHandler. It drops the
@@ -52,7 +52,7 @@ public class LossyRetryInvocationHandler
if (retryCount < this.numToDrop) {
RetryCount.set(++retryCount);
LOG.info("Drop the response. Current retryCount == " + retryCount);
- throw new UnknownHostException("Fake Exception");
+ throw new RetriableException("Fake Exception");
} else {
LOG.info("retryCount == " + retryCount
+ ". It's time to normally process the response");
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java?rev=1540239&r1=1540238&r2=1540239&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
Sat Nov 9 01:34:22 2013
@@ -558,27 +558,25 @@ public class RetryPolicies {
isWrappedStandbyException(e)) {
return new RetryAction(RetryAction.RetryDecision.FAILOVER_AND_RETRY,
getFailoverOrRetrySleepTime(failovers));
- } else if (e instanceof SocketException ||
- (e instanceof IOException && !(e instanceof
RemoteException))) {
+ } else if (e instanceof RetriableException
+ || getWrappedRetriableException(e) != null) {
+ // RetriableException or RetriableException wrapped
+ return new RetryAction(RetryAction.RetryDecision.RETRY,
+ getFailoverOrRetrySleepTime(retries));
+ } else if (e instanceof SocketException
+ || (e instanceof IOException && !(e instanceof RemoteException))) {
if (isIdempotentOrAtMostOnce) {
return RetryAction.FAILOVER_AND_RETRY;
} else {
return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
- "the invoked method is not idempotent, and unable to determine "
+
- "whether it was invoked");
+ "the invoked method is not idempotent, and unable to determine "
+ + "whether it was invoked");
}
} else {
- RetriableException re = getWrappedRetriableException(e);
- if (re != null) {
- return new RetryAction(RetryAction.RetryDecision.RETRY,
- getFailoverOrRetrySleepTime(retries));
- } else {
return fallbackPolicy.shouldRetry(e, retries, failovers,
isIdempotentOrAtMostOnce);
- }
}
}
-
}
/**
