[19/50] hadoop git commit: HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

2018-03-13 Thread aengineer 
HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afe1a3cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afe1a3cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afe1a3cc

Branch: refs/heads/HDFS-7240
Commit: afe1a3ccd56a12fec900360a8a2855c080728e65
Parents: ba0da27
Author: Inigo Goiri 
Authored: Fri Mar 9 17:18:51 2018 -0800
Committer: Inigo Goiri 
Committed: Fri Mar 9 17:18:51 2018 -0800

--
 .../federation/resolver/MountTableResolver.java | 15 +--
 .../resolver/TestMountTableResolver.java| 46 
 2 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/afe1a3cc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index dac6f7f..2c7d1f8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -238,9 +238,17 @@ public class MountTableResolver
   Entry entry = it.next();
   PathLocation loc = entry.getValue();
   String src = loc.getSourcePath();
-  if (src.startsWith(path)) {
-LOG.debug("Removing {}", src);
-it.remove();
+  if (src != null) {
+if (src.startsWith(path)) {
+  LOG.debug("Removing {}", src);
+  it.remove();
+}
+  } else {
+String dest = loc.getDefaultLocation().getDest();
+if (dest.startsWith(path)) {
+  LOG.debug("Removing default cache {}", dest);
+  it.remove();
+}
   }
 }
 
@@ -287,6 +295,7 @@ public class MountTableResolver
 if (!oldEntries.contains(srcPath)) {
   // Add node, it does not exist
   this.tree.put(srcPath, entry);
+  invalidateLocationCache(srcPath);
   LOG.info("Added new mount point {} to resolver", srcPath);
 } else {
   // Node exists, check for updates

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afe1a3cc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index a09daf0..f530fe9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.resolver;
 
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -82,6 +83,7 @@ public class TestMountTableResolver {
 Configuration conf = new Configuration();
 conf.setInt(
 FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
+conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
 mountTable = new MountTableResolver(conf);
 
 // Root mount point
@@ -479,4 +481,48 @@ public class TestMountTableResolver {
 long cacheSize = mountTable.getCacheSize();
 assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
   }
+
+  @Test
+  public void testLocationCache() throws Exception {
+List entries = new ArrayList<>();
+
+// Add entry and test location cache
+Map map1 = getMountTableEntry("1", "/testlocationcache");
+MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
+entries.add(entry1);
+
+Map map2 = getMountTableEntry("2",
+"/anothertestlocationcache");
+MountTable entry2 = MountTable.newInstance("/anothertestlocationcache",
+map2);
+entries.add(entry2);
+mountTable.refreshEntries(entries);
+

[21/50] hadoop git commit: HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

2018-03-13 Thread arp 
HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afe1a3cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afe1a3cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afe1a3cc

Branch: refs/heads/HDFS-12996
Commit: afe1a3ccd56a12fec900360a8a2855c080728e65
Parents: ba0da27
Author: Inigo Goiri 
Authored: Fri Mar 9 17:18:51 2018 -0800
Committer: Inigo Goiri 
Committed: Fri Mar 9 17:18:51 2018 -0800

--
 .../federation/resolver/MountTableResolver.java | 15 +--
 .../resolver/TestMountTableResolver.java| 46 
 2 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/afe1a3cc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index dac6f7f..2c7d1f8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -238,9 +238,17 @@ public class MountTableResolver
   Entry entry = it.next();
   PathLocation loc = entry.getValue();
   String src = loc.getSourcePath();
-  if (src.startsWith(path)) {
-LOG.debug("Removing {}", src);
-it.remove();
+  if (src != null) {
+if (src.startsWith(path)) {
+  LOG.debug("Removing {}", src);
+  it.remove();
+}
+  } else {
+String dest = loc.getDefaultLocation().getDest();
+if (dest.startsWith(path)) {
+  LOG.debug("Removing default cache {}", dest);
+  it.remove();
+}
   }
 }
 
@@ -287,6 +295,7 @@ public class MountTableResolver
 if (!oldEntries.contains(srcPath)) {
   // Add node, it does not exist
   this.tree.put(srcPath, entry);
+  invalidateLocationCache(srcPath);
   LOG.info("Added new mount point {} to resolver", srcPath);
 } else {
   // Node exists, check for updates

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afe1a3cc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index a09daf0..f530fe9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.resolver;
 
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -82,6 +83,7 @@ public class TestMountTableResolver {
 Configuration conf = new Configuration();
 conf.setInt(
 FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
+conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
 mountTable = new MountTableResolver(conf);
 
 // Root mount point
@@ -479,4 +481,48 @@ public class TestMountTableResolver {
 long cacheSize = mountTable.getCacheSize();
 assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
   }
+
+  @Test
+  public void testLocationCache() throws Exception {
+List entries = new ArrayList<>();
+
+// Add entry and test location cache
+Map map1 = getMountTableEntry("1", "/testlocationcache");
+MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
+entries.add(entry1);
+
+Map map2 = getMountTableEntry("2",
+"/anothertestlocationcache");
+MountTable entry2 = MountTable.newInstance("/anothertestlocationcache",
+map2);
+entries.add(entry2);
+mountTable.refreshEntries(entries);
+

hadoop git commit: HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

2018-03-09 Thread inigoiri 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 ba7b103c9 -> 40682baae


HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

(cherry picked from commit afe1a3ccd56a12fec900360a8a2855c080728e65)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40682baa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40682baa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40682baa

Branch: refs/heads/branch-2.9
Commit: 40682baae7e2f6f44babfa13701432ab288ead15
Parents: ba7b103
Author: Inigo Goiri 
Authored: Fri Mar 9 17:18:51 2018 -0800
Committer: Inigo Goiri 
Committed: Fri Mar 9 17:21:43 2018 -0800

--
 .../federation/resolver/MountTableResolver.java | 15 +--
 .../resolver/TestMountTableResolver.java| 46 
 2 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/40682baa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index dac6f7f..2c7d1f8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -238,9 +238,17 @@ public class MountTableResolver
   Entry entry = it.next();
   PathLocation loc = entry.getValue();
   String src = loc.getSourcePath();
-  if (src.startsWith(path)) {
-LOG.debug("Removing {}", src);
-it.remove();
+  if (src != null) {
+if (src.startsWith(path)) {
+  LOG.debug("Removing {}", src);
+  it.remove();
+}
+  } else {
+String dest = loc.getDefaultLocation().getDest();
+if (dest.startsWith(path)) {
+  LOG.debug("Removing default cache {}", dest);
+  it.remove();
+}
   }
 }
 
@@ -287,6 +295,7 @@ public class MountTableResolver
 if (!oldEntries.contains(srcPath)) {
   // Add node, it does not exist
   this.tree.put(srcPath, entry);
+  invalidateLocationCache(srcPath);
   LOG.info("Added new mount point {} to resolver", srcPath);
 } else {
   // Node exists, check for updates

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40682baa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index a09daf0..f530fe9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.resolver;
 
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -82,6 +83,7 @@ public class TestMountTableResolver {
 Configuration conf = new Configuration();
 conf.setInt(
 FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
+conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
 mountTable = new MountTableResolver(conf);
 
 // Root mount point
@@ -479,4 +481,48 @@ public class TestMountTableResolver {
 long cacheSize = mountTable.getCacheSize();
 assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
   }
+
+  @Test
+  public void testLocationCache() throws Exception {
+List entries = new ArrayList<>();
+
+// Add entry and test location cache
+Map map1 = getMountTableEntry("1", "/testlocationcache");
+MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
+entries.add(entry1);
+
+Map map2 = getMountTableEntry("2",
+"/anothertestlocationcache");
+

hadoop git commit: HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

2018-03-09 Thread inigoiri 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 121daaf47 -> 4b6b07690


HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

(cherry picked from commit afe1a3ccd56a12fec900360a8a2855c080728e65)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b6b0769
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b6b0769
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b6b0769

Branch: refs/heads/branch-2
Commit: 4b6b07690e7d4f582b9328017e3204b700a4c1eb
Parents: 121daaf
Author: Inigo Goiri 
Authored: Fri Mar 9 17:18:51 2018 -0800
Committer: Inigo Goiri 
Committed: Fri Mar 9 17:21:10 2018 -0800

--
 .../federation/resolver/MountTableResolver.java | 15 +--
 .../resolver/TestMountTableResolver.java| 46 
 2 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b6b0769/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index dac6f7f..2c7d1f8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -238,9 +238,17 @@ public class MountTableResolver
   Entry entry = it.next();
   PathLocation loc = entry.getValue();
   String src = loc.getSourcePath();
-  if (src.startsWith(path)) {
-LOG.debug("Removing {}", src);
-it.remove();
+  if (src != null) {
+if (src.startsWith(path)) {
+  LOG.debug("Removing {}", src);
+  it.remove();
+}
+  } else {
+String dest = loc.getDefaultLocation().getDest();
+if (dest.startsWith(path)) {
+  LOG.debug("Removing default cache {}", dest);
+  it.remove();
+}
   }
 }
 
@@ -287,6 +295,7 @@ public class MountTableResolver
 if (!oldEntries.contains(srcPath)) {
   // Add node, it does not exist
   this.tree.put(srcPath, entry);
+  invalidateLocationCache(srcPath);
   LOG.info("Added new mount point {} to resolver", srcPath);
 } else {
   // Node exists, check for updates

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b6b0769/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index a09daf0..f530fe9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.resolver;
 
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -82,6 +83,7 @@ public class TestMountTableResolver {
 Configuration conf = new Configuration();
 conf.setInt(
 FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
+conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
 mountTable = new MountTableResolver(conf);
 
 // Root mount point
@@ -479,4 +481,48 @@ public class TestMountTableResolver {
 long cacheSize = mountTable.getCacheSize();
 assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
   }
+
+  @Test
+  public void testLocationCache() throws Exception {
+List entries = new ArrayList<>();
+
+// Add entry and test location cache
+Map map1 = getMountTableEntry("1", "/testlocationcache");
+MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
+entries.add(entry1);
+
+Map map2 = getMountTableEntry("2",
+"/anothertestlocationcache");
+MountTable

hadoop git commit: HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

2018-03-09 Thread inigoiri 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 bd2ebf031 -> c453fcb23


HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

(cherry picked from commit afe1a3ccd56a12fec900360a8a2855c080728e65)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c453fcb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c453fcb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c453fcb2

Branch: refs/heads/branch-3.0
Commit: c453fcb23fde0153cd8ebfa808c5728a2c8b99d4
Parents: bd2ebf0
Author: Inigo Goiri 
Authored: Fri Mar 9 17:18:51 2018 -0800
Committer: Inigo Goiri 
Committed: Fri Mar 9 17:20:29 2018 -0800

--
 .../federation/resolver/MountTableResolver.java | 15 +--
 .../resolver/TestMountTableResolver.java| 46 
 2 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c453fcb2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index dac6f7f..2c7d1f8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -238,9 +238,17 @@ public class MountTableResolver
   Entry entry = it.next();
   PathLocation loc = entry.getValue();
   String src = loc.getSourcePath();
-  if (src.startsWith(path)) {
-LOG.debug("Removing {}", src);
-it.remove();
+  if (src != null) {
+if (src.startsWith(path)) {
+  LOG.debug("Removing {}", src);
+  it.remove();
+}
+  } else {
+String dest = loc.getDefaultLocation().getDest();
+if (dest.startsWith(path)) {
+  LOG.debug("Removing default cache {}", dest);
+  it.remove();
+}
   }
 }
 
@@ -287,6 +295,7 @@ public class MountTableResolver
 if (!oldEntries.contains(srcPath)) {
   // Add node, it does not exist
   this.tree.put(srcPath, entry);
+  invalidateLocationCache(srcPath);
   LOG.info("Added new mount point {} to resolver", srcPath);
 } else {
   // Node exists, check for updates

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c453fcb2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index a09daf0..f530fe9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.resolver;
 
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -82,6 +83,7 @@ public class TestMountTableResolver {
 Configuration conf = new Configuration();
 conf.setInt(
 FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
+conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
 mountTable = new MountTableResolver(conf);
 
 // Root mount point
@@ -479,4 +481,48 @@ public class TestMountTableResolver {
 long cacheSize = mountTable.getCacheSize();
 assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
   }
+
+  @Test
+  public void testLocationCache() throws Exception {
+List entries = new ArrayList<>();
+
+// Add entry and test location cache
+Map map1 = getMountTableEntry("1", "/testlocationcache");
+MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
+entries.add(entry1);
+
+Map map2 = getMountTableEntry("2",
+"/anothertestlocationcache");
+

hadoop git commit: HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

2018-03-09 Thread inigoiri 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 c337c97c6 -> 4d41f933a


HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

(cherry picked from commit afe1a3ccd56a12fec900360a8a2855c080728e65)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d41f933
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d41f933
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d41f933

Branch: refs/heads/branch-3.1
Commit: 4d41f933a298a2644284abcb4971c30543e85dc3
Parents: c337c97
Author: Inigo Goiri 
Authored: Fri Mar 9 17:18:51 2018 -0800
Committer: Inigo Goiri 
Committed: Fri Mar 9 17:19:50 2018 -0800

--
 .../federation/resolver/MountTableResolver.java | 15 +--
 .../resolver/TestMountTableResolver.java| 46 
 2 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d41f933/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index dac6f7f..2c7d1f8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -238,9 +238,17 @@ public class MountTableResolver
   Entry entry = it.next();
   PathLocation loc = entry.getValue();
   String src = loc.getSourcePath();
-  if (src.startsWith(path)) {
-LOG.debug("Removing {}", src);
-it.remove();
+  if (src != null) {
+if (src.startsWith(path)) {
+  LOG.debug("Removing {}", src);
+  it.remove();
+}
+  } else {
+String dest = loc.getDefaultLocation().getDest();
+if (dest.startsWith(path)) {
+  LOG.debug("Removing default cache {}", dest);
+  it.remove();
+}
   }
 }
 
@@ -287,6 +295,7 @@ public class MountTableResolver
 if (!oldEntries.contains(srcPath)) {
   // Add node, it does not exist
   this.tree.put(srcPath, entry);
+  invalidateLocationCache(srcPath);
   LOG.info("Added new mount point {} to resolver", srcPath);
 } else {
   // Node exists, check for updates

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d41f933/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index a09daf0..f530fe9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.resolver;
 
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -82,6 +83,7 @@ public class TestMountTableResolver {
 Configuration conf = new Configuration();
 conf.setInt(
 FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
+conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
 mountTable = new MountTableResolver(conf);
 
 // Root mount point
@@ -479,4 +481,48 @@ public class TestMountTableResolver {
 long cacheSize = mountTable.getCacheSize();
 assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
   }
+
+  @Test
+  public void testLocationCache() throws Exception {
+List entries = new ArrayList<>();
+
+// Add entry and test location cache
+Map map1 = getMountTableEntry("1", "/testlocationcache");
+MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
+entries.add(entry1);
+
+Map map2 = getMountTableEntry("2",
+"/anothertestlocationcache");
+

hadoop git commit: HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

2018-03-09 Thread inigoiri 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ba0da2785 -> afe1a3ccd


HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afe1a3cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afe1a3cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afe1a3cc

Branch: refs/heads/trunk
Commit: afe1a3ccd56a12fec900360a8a2855c080728e65
Parents: ba0da27
Author: Inigo Goiri 
Authored: Fri Mar 9 17:18:51 2018 -0800
Committer: Inigo Goiri 
Committed: Fri Mar 9 17:18:51 2018 -0800

--
 .../federation/resolver/MountTableResolver.java | 15 +--
 .../resolver/TestMountTableResolver.java| 46 
 2 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/afe1a3cc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index dac6f7f..2c7d1f8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -238,9 +238,17 @@ public class MountTableResolver
   Entry entry = it.next();
   PathLocation loc = entry.getValue();
   String src = loc.getSourcePath();
-  if (src.startsWith(path)) {
-LOG.debug("Removing {}", src);
-it.remove();
+  if (src != null) {
+if (src.startsWith(path)) {
+  LOG.debug("Removing {}", src);
+  it.remove();
+}
+  } else {
+String dest = loc.getDefaultLocation().getDest();
+if (dest.startsWith(path)) {
+  LOG.debug("Removing default cache {}", dest);
+  it.remove();
+}
   }
 }
 
@@ -287,6 +295,7 @@ public class MountTableResolver
 if (!oldEntries.contains(srcPath)) {
   // Add node, it does not exist
   this.tree.put(srcPath, entry);
+  invalidateLocationCache(srcPath);
   LOG.info("Added new mount point {} to resolver", srcPath);
 } else {
   // Node exists, check for updates

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afe1a3cc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index a09daf0..f530fe9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.resolver;
 
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -82,6 +83,7 @@ public class TestMountTableResolver {
 Configuration conf = new Configuration();
 conf.setInt(
 FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
+conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
 mountTable = new MountTableResolver(conf);
 
 // Root mount point
@@ -479,4 +481,48 @@ public class TestMountTableResolver {
 long cacheSize = mountTable.getCacheSize();
 assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
   }
+
+  @Test
+  public void testLocationCache() throws Exception {
+List entries = new ArrayList<>();
+
+// Add entry and test location cache
+Map map1 = getMountTableEntry("1", "/testlocationcache");
+MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
+entries.add(entry1);
+
+Map map2 = getMountTableEntry("2",
+"/anothertestlocationcache");
+MountTable entry2 = MountTable.newInstance("/anothertestlocationcache",
+