[41/44] hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.

2018-04-13 Thread xyao 
HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor 
Bota.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e66e287e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e66e287e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e66e287e

Branch: refs/heads/HDFS-7240
Commit: e66e287efe2b43e710137a628f03c7df3ebdf498
Parents: 6503593
Author: Wei-Chiu Chuang 
Authored: Fri Apr 13 09:17:34 2018 -0700
Committer: Wei-Chiu Chuang 
Committed: Fri Apr 13 09:17:34 2018 -0700

--
 .../hdfs/shortcircuit/ShortCircuitCache.java| 11 ++---
 .../shortcircuit/TestShortCircuitCache.java | 26 
 2 files changed, 33 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index b26652b..c2f0350 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable {
 unref(replica);
   }
 
+  static final int FETCH_OR_CREATE_RETRY_TIMES = 3;
   /**
* Fetch or create a replica.
*
@@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable {
*/
   public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key,
   ShortCircuitReplicaCreator creator) {
-Waitable newWaitable = null;
+Waitable newWaitable;
 lock.lock();
 try {
   ShortCircuitReplicaInfo info = null;
-  do {
+  for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){
 if (closed) {
   LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.",
   this, key);
@@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable {
 if (waitable != null) {
   try {
 info = fetch(key, waitable);
+break;
   } catch (RetriableException e) {
 LOG.debug("{}: retrying {}", this, e.getMessage());
   }
 }
-  } while (false);
+  }
   if (info != null) return info;
   // We need to load the replica ourselves.
   newWaitable = new Waitable<>(lock.newCondition());
@@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable {
*
* @throws RetriableException   If the caller needs to retry.
*/
-  private ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
+  @VisibleForTesting // ONLY for testing
+  protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
   Waitable waitable) throws RetriableException {
 // Another thread is already in the process of loading this
 // ShortCircuitReplica.  So we simply wait for it to complete.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index 7ba0edc..5da6a25 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -65,6 +65,7 @@ import 
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RetriableException;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -793,4 +794,29 @@ public class TestShortCircuitCache {
 cluster.shutdown();
 sockDir.close();
   }
+
+  @Test
+  public void testFetchOrCreateRetries() throws Exception {
+try(ShortCircuitCache cache = Mockito
+.spy(new ShortCircuitCache(10, 1000, 10, 1000, 1, 1, 0))) {
+  final TestFileDescriptorPair pair

hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.

2018-04-13 Thread weichiu 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 bce57a139 -> dc01e323e


HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor 
Bota.

(cherry picked from commit e66e287efe2b43e710137a628f03c7df3ebdf498)
(cherry picked from commit ca8bb322be5e21ce00bb49cecfc4f515b49d2a37)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc01e323
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc01e323
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc01e323

Branch: refs/heads/branch-3.0
Commit: dc01e323ed24c22f9db88af4963654a037f72e75
Parents: bce57a1
Author: Wei-Chiu Chuang 
Authored: Fri Apr 13 09:17:34 2018 -0700
Committer: Wei-Chiu Chuang 
Committed: Fri Apr 13 12:20:01 2018 -0700

--
 .../hdfs/shortcircuit/ShortCircuitCache.java| 11 ++---
 .../shortcircuit/TestShortCircuitCache.java | 26 
 2 files changed, 33 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc01e323/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index b26652b..c2f0350 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable {
 unref(replica);
   }
 
+  static final int FETCH_OR_CREATE_RETRY_TIMES = 3;
   /**
* Fetch or create a replica.
*
@@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable {
*/
   public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key,
   ShortCircuitReplicaCreator creator) {
-Waitable newWaitable = null;
+Waitable newWaitable;
 lock.lock();
 try {
   ShortCircuitReplicaInfo info = null;
-  do {
+  for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){
 if (closed) {
   LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.",
   this, key);
@@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable {
 if (waitable != null) {
   try {
 info = fetch(key, waitable);
+break;
   } catch (RetriableException e) {
 LOG.debug("{}: retrying {}", this, e.getMessage());
   }
 }
-  } while (false);
+  }
   if (info != null) return info;
   // We need to load the replica ourselves.
   newWaitable = new Waitable<>(lock.newCondition());
@@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable {
*
* @throws RetriableException   If the caller needs to retry.
*/
-  private ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
+  @VisibleForTesting // ONLY for testing
+  protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
   Waitable waitable) throws RetriableException {
 // Another thread is already in the process of loading this
 // ShortCircuitReplica.  So we simply wait for it to complete.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc01e323/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index 7ba0edc..5da6a25 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -65,6 +65,7 @@ import 
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RetriableException;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -793,4 +794,29 @@ public class TestShortCircuitCache {
 cluster.shutdown();
 sockDir.close();
   }
+
+  @Test

hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.

2018-04-13 Thread weichiu 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 994c7d66e -> ca8bb322b


HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor 
Bota.

(cherry picked from commit e66e287efe2b43e710137a628f03c7df3ebdf498)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca8bb322
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca8bb322
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca8bb322

Branch: refs/heads/branch-3.1
Commit: ca8bb322be5e21ce00bb49cecfc4f515b49d2a37
Parents: 994c7d6
Author: Wei-Chiu Chuang 
Authored: Fri Apr 13 09:17:34 2018 -0700
Committer: Wei-Chiu Chuang 
Committed: Fri Apr 13 12:18:49 2018 -0700

--
 .../hdfs/shortcircuit/ShortCircuitCache.java| 11 ++---
 .../shortcircuit/TestShortCircuitCache.java | 26 
 2 files changed, 33 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8bb322/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index b26652b..c2f0350 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable {
 unref(replica);
   }
 
+  static final int FETCH_OR_CREATE_RETRY_TIMES = 3;
   /**
* Fetch or create a replica.
*
@@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable {
*/
   public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key,
   ShortCircuitReplicaCreator creator) {
-Waitable newWaitable = null;
+Waitable newWaitable;
 lock.lock();
 try {
   ShortCircuitReplicaInfo info = null;
-  do {
+  for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){
 if (closed) {
   LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.",
   this, key);
@@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable {
 if (waitable != null) {
   try {
 info = fetch(key, waitable);
+break;
   } catch (RetriableException e) {
 LOG.debug("{}: retrying {}", this, e.getMessage());
   }
 }
-  } while (false);
+  }
   if (info != null) return info;
   // We need to load the replica ourselves.
   newWaitable = new Waitable<>(lock.newCondition());
@@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable {
*
* @throws RetriableException   If the caller needs to retry.
*/
-  private ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
+  @VisibleForTesting // ONLY for testing
+  protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
   Waitable waitable) throws RetriableException {
 // Another thread is already in the process of loading this
 // ShortCircuitReplica.  So we simply wait for it to complete.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8bb322/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index 7ba0edc..5da6a25 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -65,6 +65,7 @@ import 
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RetriableException;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -793,4 +794,29 @@ public class TestShortCircuitCache {
 cluster.shutdown();
 sockDir.close();
   }
+
+  @Test
+  public void testFetchOrCreateRetries() throws Exception {
+tr

hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.

2018-04-13 Thread weichiu 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 650359371 -> e66e287ef


HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor 
Bota.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e66e287e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e66e287e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e66e287e

Branch: refs/heads/trunk
Commit: e66e287efe2b43e710137a628f03c7df3ebdf498
Parents: 6503593
Author: Wei-Chiu Chuang 
Authored: Fri Apr 13 09:17:34 2018 -0700
Committer: Wei-Chiu Chuang 
Committed: Fri Apr 13 09:17:34 2018 -0700

--
 .../hdfs/shortcircuit/ShortCircuitCache.java| 11 ++---
 .../shortcircuit/TestShortCircuitCache.java | 26 
 2 files changed, 33 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index b26652b..c2f0350 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable {
 unref(replica);
   }
 
+  static final int FETCH_OR_CREATE_RETRY_TIMES = 3;
   /**
* Fetch or create a replica.
*
@@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable {
*/
   public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key,
   ShortCircuitReplicaCreator creator) {
-Waitable newWaitable = null;
+Waitable newWaitable;
 lock.lock();
 try {
   ShortCircuitReplicaInfo info = null;
-  do {
+  for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){
 if (closed) {
   LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.",
   this, key);
@@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable {
 if (waitable != null) {
   try {
 info = fetch(key, waitable);
+break;
   } catch (RetriableException e) {
 LOG.debug("{}: retrying {}", this, e.getMessage());
   }
 }
-  } while (false);
+  }
   if (info != null) return info;
   // We need to load the replica ourselves.
   newWaitable = new Waitable<>(lock.newCondition());
@@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable {
*
* @throws RetriableException   If the caller needs to retry.
*/
-  private ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
+  @VisibleForTesting // ONLY for testing
+  protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
   Waitable waitable) throws RetriableException {
 // Another thread is already in the process of loading this
 // ShortCircuitReplica.  So we simply wait for it to complete.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index 7ba0edc..5da6a25 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -65,6 +65,7 @@ import 
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RetriableException;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -793,4 +794,29 @@ public class TestShortCircuitCache {
 cluster.shutdown();
 sockDir.close();
   }
+
+  @Test
+  public void testFetchOrCreateRetries() throws Exception {
+try(ShortCircuitCache cache = Mockito
+.spy(new ShortCircuitCache(10, 1000