Repository: hadoop Updated Branches: refs/heads/branch-2 85473cd61 -> b00b216a9
YARN-3205. FileSystemRMStateStore should disable FileSystem Cache to avoid get a Filesystem with an old configuration. Contributed by Zhihai Xu. (cherry picked from commit 3bc72cc16d8c7b8addd8f565523001dfcc32b891) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b00b216a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b00b216a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b00b216a Branch: refs/heads/branch-2 Commit: b00b216a919417b2abd364d6433ad08fad7db134 Parents: 85473cd Author: Tsuyoshi Ozawa <[email protected]> Authored: Wed Mar 18 11:53:14 2015 +0900 Committer: Tsuyoshi Ozawa <[email protected]> Committed: Wed Mar 18 11:53:40 2015 +0900 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../recovery/FileSystemRMStateStore.java | 22 +++++++++++++++----- .../recovery/TestFSRMStateStore.java | 5 +++++ 3 files changed, 25 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00b216a/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 69b3c43..4b81d04 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -24,6 +24,9 @@ Release 2.8.0 - UNRELEASED YARN-3305. Normalize AM resource request on app submission. (Rohith Sharmaks via jianhe) + YARN-3205 FileSystemRMStateStore should disable FileSystem Cache to avoid + get a Filesystem with an old configuration. (Zhihai Xu via ozawa) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00b216a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 8147597..7652a07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -84,7 +84,10 @@ public class FileSystemRMStateStore extends RMStateStore { protected static final String AMRMTOKEN_SECRET_MANAGER_NODE = "AMRMTokenSecretManagerNode"; + @VisibleForTesting protected FileSystem fs; + @VisibleForTesting + protected Configuration fsConf; private Path rootDirPath; @Private @@ -121,14 +124,23 @@ public class FileSystemRMStateStore extends RMStateStore { // create filesystem only now, as part of service-start. By this time, RM is // authenticated with kerberos so we are good to create a file-system // handle. - Configuration conf = new Configuration(getConfig()); - conf.setBoolean("dfs.client.retry.policy.enabled", true); + fsConf = new Configuration(getConfig()); + fsConf.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = - conf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, + fsConf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC); - conf.set("dfs.client.retry.policy.spec", retryPolicy); + fsConf.set("dfs.client.retry.policy.spec", retryPolicy); + + String scheme = fsWorkingPath.toUri().getScheme(); + if (scheme == null) { + scheme = FileSystem.getDefaultUri(fsConf).getScheme(); + } + if (scheme != null) { + String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme); + fsConf.setBoolean(disableCacheName, true); + } - fs = fsWorkingPath.getFileSystem(conf); + fs = fsWorkingPath.getFileSystem(fsConf); mkdirsWithRetries(rmDTSecretManagerRoot); mkdirsWithRetries(rmAppRoot); mkdirsWithRetries(amrmTokenSecretManagerRoot); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00b216a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index d2eddd6..c5f5578 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -106,6 +106,11 @@ public class TestFSRMStateStore extends RMStateStoreTestBase { this.store = new TestFileSystemRMStore(conf); Assert.assertEquals(store.getNumRetries(), 8); Assert.assertEquals(store.getRetryInterval(), 900L); + Assert.assertTrue(store.fs.getConf() == store.fsConf); + FileSystem previousFs = store.fs; + store.startInternal(); + Assert.assertTrue(store.fs != previousFs); + Assert.assertTrue(store.fs.getConf() == store.fsConf); return store; }
