Repository: hadoop Updated Branches: refs/heads/trunk 319defafc -> 7fab787de
HDFS-13253. RBF: Quota management incorrect parent-child relationship judgement. Contributed by Yiqun Lin. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7fab787d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7fab787d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7fab787d Branch: refs/heads/trunk Commit: 7fab787de72756863a91c2358da5c611afdb80e9 Parents: 319defa Author: Yiqun Lin <yq...@apache.org> Authored: Tue Mar 13 10:30:20 2018 +0800 Committer: Yiqun Lin <yq...@apache.org> Committed: Tue Mar 13 10:30:20 2018 +0800 ---------------------------------------------------------------------- .../federation/resolver/MountTableResolver.java | 14 ++------------ .../federation/router/FederationUtil.java | 20 ++++++++++++++++++++ .../federation/router/RouterQuotaManager.java | 14 +++++++++++++- .../router/TestRouterQuotaManager.java | 12 ++++++++++++ 4 files changed, 47 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fab787d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java index 2c7d1f8..27b43e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.federation.resolver; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE; import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE; import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT; +import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry; import java.io.IOException; import java.util.Collection; @@ -239,7 +240,7 @@ public class MountTableResolver PathLocation loc = entry.getValue(); String src = loc.getSourcePath(); if (src != null) { - if (src.startsWith(path)) { + if(isParentEntry(src, path)) { LOG.debug("Removing {}", src); it.remove(); } @@ -530,17 +531,6 @@ public class MountTableResolver return this.defaultNameService; } - private boolean isParentEntry(final String path, final String parent) { - if (!path.startsWith(parent)) { - return false; - } - if (path.equals(parent)) { - return true; - } - return path.charAt(parent.length()) == Path.SEPARATOR_CHAR - || parent.equals(Path.SEPARATOR); - } - /** * Find the deepest mount point for a path. * @param path Path to look for. http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fab787d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java index 8d631e9..3dfd998 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java @@ -26,6 +26,7 @@ import java.net.URL; import java.net.URLConnection; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; @@ -186,4 +187,23 @@ public final class FederationUtil { ActiveNamenodeResolver.class); return newInstance(conf, stateStore, StateStoreService.class, clazz); } + + /** + * Check if the given path is the child of parent path. + * @param path Path to be check. + * @param parent Parent path. + * @return True if parent path is parent entry for given path. + */ + public static boolean isParentEntry(final String path, final String parent) { + if (!path.startsWith(parent)) { + return false; + } + + if (path.equals(parent)) { + return true; + } + + return path.charAt(parent.length()) == Path.SEPARATOR_CHAR + || parent.equals(Path.SEPARATOR); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fab787d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java index fc3575c..0df34fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry; + +import java.util.HashSet; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; @@ -94,7 +97,16 @@ public class RouterQuotaManager { String from = parentPath; String to = parentPath + Character.MAX_VALUE; SortedMap<String, RouterQuotaUsage> subMap = this.cache.subMap(from, to); - return subMap.keySet(); + + Set<String> validPaths = new HashSet<>(); + if (subMap != null) { + for (String path : subMap.keySet()) { + if (isParentEntry(path, parentPath)) { + validPaths.add(path); + } + } + } + return validPaths; } finally { readLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fab787d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java index 346c881..ce3ee17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java @@ -57,6 +57,18 @@ public class TestRouterQuotaManager { assertTrue(childrenPaths.contains("/path1/subdir") && childrenPaths.contains("/path1/subdir/subdir") && childrenPaths.contains("/path1")); + + // test for corner case + manager.put("/path3", quotaUsage); + manager.put("/path3/subdir", quotaUsage); + manager.put("/path3-subdir", quotaUsage); + + childrenPaths = manager.getPaths("/path3"); + assertEquals(2, childrenPaths.size()); + // path /path3-subdir should not be returned + assertTrue(childrenPaths.contains("/path3") + && childrenPaths.contains("/path3/subdir") + && !childrenPaths.contains("/path3-subdir")); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
