This is an automated email from the ASF dual-hosted git repository. eyang pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 155864d YARN-9875. Improve fair scheduler configuration store on HDFS. Contributed by Prabhu Joseph 155864d is described below commit 155864da006346a500ff35c2f6b69281093195b1 Author: Eric Yang <ey...@apache.org> AuthorDate: Fri Oct 18 20:30:11 2019 -0400 YARN-9875. Improve fair scheduler configuration store on HDFS. Contributed by Prabhu Joseph --- .../conf/FSSchedulerConfigurationStore.java | 12 ++++- .../conf/TestFSSchedulerConfigurationStore.java | 51 ++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java index 464ef14..855939e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java @@ -66,7 +66,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { private Path configVersionFile; @Override - public void initialize(Configuration conf, Configuration vSchedConf, + public void initialize(Configuration fsConf, Configuration vSchedConf, RMContext rmContext) throws Exception { this.configFilePathFilter = new PathFilter() { @Override @@ -80,6 +80,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { } }; + Configuration conf = new Configuration(fsConf); String schedulerConfPathStr = conf.get( YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH); if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) { @@ -88,6 +89,15 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { + " must be set"); } this.schedulerConfDir = new Path(schedulerConfPathStr); + String scheme = schedulerConfDir.toUri().getScheme(); + if (scheme == null) { + scheme = FileSystem.getDefaultUri(conf).getScheme(); + } + if (scheme != null) { + String disableCacheName = String.format("fs.%s.impl.disable.cache", + scheme); + conf.setBoolean(disableCacheName, true); + } this.fileSystem = this.schedulerConfDir.getFileSystem(conf); this.maxVersion = conf.getInt( YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java index 33596c3..7968372 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java @@ -28,6 +28,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.junit.After; @@ -138,6 +140,55 @@ public class TestFSSchedulerConfigurationStore { } @Test + public void testFileSystemClose() throws Exception { + MiniDFSCluster hdfsCluster = null; + FileSystem fs = null; + Path path = new Path("/tmp/confstore"); + try { + HdfsConfiguration hdfsConfig = new HdfsConfiguration(); + hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) + .numDataNodes(1).build(); + + fs = hdfsCluster.getFileSystem(); + if (!fs.exists(path)) { + fs.mkdirs(path); + } + + FSSchedulerConfigurationStore configStore = + new FSSchedulerConfigurationStore(); + hdfsConfig.set(YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH, + path.toString()); + configStore.initialize(hdfsConfig, hdfsConfig, null); + + // Close the FileSystem object and validate + fs.close(); + + try { + Map<String, String> updates = new HashMap<>(); + updates.put("testkey", "testvalue"); + LogMutation logMutation = new LogMutation(updates, "test"); + configStore.logMutation(logMutation); + configStore.confirmMutation(true); + } catch (IOException e) { + if (e.getMessage().contains("Filesystem closed")) { + fail("FSSchedulerConfigurationStore failed to handle " + + "FileSystem close"); + } else { + fail("Should not get any exceptions"); + } + } + } finally { + fs = hdfsCluster.getFileSystem(); + if (fs.exists(path)) { + fs.delete(path, true); + } + if (hdfsCluster != null) { + hdfsCluster.shutdown(); + } + } + } + + @Test public void testFormatConfiguration() throws Exception { Configuration schedulerConf = new Configuration(); schedulerConf.set("a", "a"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org