This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3990ffa YARN-9909. Offline format of YarnConfigurationStore.
Contributed by Prabhu Joseph.
3990ffa is described below
commit 3990ffa0834e6b24314a565a92e92d6e8fa37a1e
Author: Sunil G <sun...@apache.org>
AuthorDate: Thu Oct 17 06:43:06 2019 +0530
YARN-9909. Offline format of YarnConfigurationStore. Contributed by Prabhu
Joseph.
---
.../server/resourcemanager/ResourceManager.java | 48 +++++++++++++++++++++-
.../conf/MutableCSConfigurationProvider.java | 21 +---------
.../conf/YarnConfigurationStoreFactory.java | 22 ++++++++--
.../resourcemanager/TestRMStoreCommands.java | 35 ++++++++++++++++
.../src/site/markdown/YarnCommands.md | 1 +
5 files changed, 102 insertions(+), 25 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 351519b..228c1b5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -105,12 +105,15 @@ import
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStoreFactory;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.MemoryPlacementConstraintManager;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManagerService;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
@@ -1563,6 +1566,8 @@ public class ResourceManager extends CompositeService
if (argv.length >= 1) {
if (argv[0].equals("-format-state-store")) {
deleteRMStateStore(conf);
+ } else if (argv[0].equals("-format-conf-store")) {
+ deleteRMConfStore(conf);
} else if (argv[0].equals("-remove-application-from-state-store")
&& argv.length == 2) {
removeApplication(conf, argv[1]);
@@ -1671,6 +1676,45 @@ public class ResourceManager extends CompositeService
}
}
+ /**
+ * Deletes the YarnConfigurationStore
+ *
+ * @param conf
+ * @throws Exception
+ */
+ @VisibleForTesting
+ static void deleteRMConfStore(Configuration conf) throws Exception {
+ ResourceManager rm = new ResourceManager();
+ rm.conf = conf;
+ ResourceScheduler scheduler = rm.createScheduler();
+ RMContextImpl rmContext = new RMContextImpl();
+ rmContext.setResourceManager(rm);
+
+ boolean isConfigurationMutable = false;
+ String confProviderStr = conf.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+ switch (confProviderStr) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
+ case YarnConfiguration.FS_CONFIGURATION_STORE:
+ isConfigurationMutable = true;
+ break;
+ default:
+ }
+
+ if (scheduler instanceof MutableConfScheduler && isConfigurationMutable) {
+ YarnConfigurationStore confStore = YarnConfigurationStoreFactory
+ .getStore(conf);
+ confStore.initialize(conf, conf, rmContext);
+ confStore.format();
+ } else {
+ System.out.println("Scheduler Configuration format only " +
+ "supported by MutableConfScheduler.");
+ }
+ }
+
@VisibleForTesting
static void removeApplication(Configuration conf, String applicationId)
throws Exception {
@@ -1691,7 +1735,9 @@ public class ResourceManager extends CompositeService
private static void printUsage(PrintStream out) {
out.println("Usage: yarn resourcemanager [-format-state-store]");
out.println(" "
- + "[-remove-application-from-state-store <appId>]" + "\n");
+ + "[-remove-application-from-state-store <appId>]");
+ out.println(" "
+ + "[-format-conf-store]" + "\n");
out.println("[-convert-fs-configuration ");
out.println(FSConfigToCSConfigConverter.WARNING_TEXT);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index ba72763..494c18c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -68,26 +68,7 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
@Override
public void init(Configuration config) throws IOException {
- String store = config.get(
- YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
- YarnConfiguration.MEMORY_CONFIGURATION_STORE);
- switch (store) {
- case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
- this.confStore = new InMemoryConfigurationStore();
- break;
- case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
- this.confStore = new LeveldbConfigurationStore();
- break;
- case YarnConfiguration.ZK_CONFIGURATION_STORE:
- this.confStore = new ZKConfigurationStore();
- break;
- case YarnConfiguration.FS_CONFIGURATION_STORE:
- this.confStore = new FSSchedulerConfigurationStore();
- break;
- default:
- this.confStore = YarnConfigurationStoreFactory.getStore(config);
- break;
- }
+ this.confStore = YarnConfigurationStoreFactory.getStore(config);
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
this.schedConf = new Configuration(false);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
index af637d4..27e573e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
@@ -37,10 +37,24 @@ public final class YarnConfigurationStoreFactory {
}
public static YarnConfigurationStore getStore(Configuration conf) {
- Class<? extends YarnConfigurationStore> storeClass =
- conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ String store = conf.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+ switch (store) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ return new InMemoryConfigurationStore();
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ return new LeveldbConfigurationStore();
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
+ return new ZKConfigurationStore();
+ case YarnConfiguration.FS_CONFIGURATION_STORE:
+ return new FSSchedulerConfigurationStore();
+ default:
+ Class<? extends YarnConfigurationStore> storeClass =
+
conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
InMemoryConfigurationStore.class, YarnConfigurationStore.class);
- LOG.info("Using YarnConfigurationStore implementation - " + storeClass);
- return ReflectionUtils.newInstance(storeClass, conf);
+ LOG.info("Using YarnConfigurationStore implementation - " +
storeClass);
+ return ReflectionUtils.newInstance(storeClass, conf);
+ }
}
}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java
index 0203351..e2c7834 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -67,6 +69,39 @@ public class TestRMStoreCommands {
}
@Test
+ public void testFormatConfStoreCmdForZK() throws Exception {
+ try (TestingServer curatorTestingServer =
+ TestZKRMStateStore.setupCuratorServer();
+ CuratorFramework curatorFramework = TestZKRMStateStore.
+ setupCuratorFramework(curatorTestingServer)) {
+ Configuration conf = TestZKRMStateStore.createHARMConf("rm1,rm2", "rm1",
+ 1234, false, curatorTestingServer);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.ZK_CONFIGURATION_STORE);
+
+ ResourceManager rm = new MockRM(conf);
+ rm.start();
+
+ String confStorePath = conf.get(
+ YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH)
+ + "/CONF_STORE";
+ assertNotNull("Failed to initialize ZKConfigurationStore",
+ curatorFramework.checkExists().forPath(confStorePath));
+
+ rm.close();
+ try {
+ ResourceManager.deleteRMConfStore(conf);
+ } catch (Exception e) {
+ fail("Exception should not be thrown during format rm conf store" +
+ " operation.");
+ }
+ assertNull("Failed to format ZKConfigurationStore",
+ curatorFramework.checkExists().forPath(confStorePath));
+ }
+ }
+
+ @Test
public void testRemoveApplicationFromStateStoreCmdForZK() throws Exception {
StateChangeRequestInfo req = new StateChangeRequestInfo(
HAServiceProtocol.RequestSource.REQUEST_BY_USER);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index c0bffa2..a772860 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -193,6 +193,7 @@ Usage: `yarn resourcemanager [-format-state-store]`
|:---- |:---- |
| -format-state-store | Formats the RMStateStore. This will clear the
RMStateStore and is useful if past applications are no longer needed. This
should be run only when the ResourceManager is not running. |
| -remove-application-from-state-store \<appId\> | Remove the application from
RMStateStore. This should be run only when the ResourceManager is not running. |
+| -format-conf-store | Formats the YarnConfigurationStore. This will clear the
persisted scheduler configuration under YarnConfigurationStore. This should be
run only when the ResourceManager is not running. |
| -convert-fs-configuration [-y|yarnsiteconfig] [-f|fsconfig]
[-r|rulesconfig] [-o|output-directory] [-p|print]
[-c|cluster-resource] | WARNING: This feature is experimental and not
intended for production use! Development is still in progress so the converter
should not be considered complete! <br/> Converts the specified Fair Scheduler
configuration to Capacity Scheduler configuration. Requires two mandatory input
files. First, the yarn-site.xml with the f [...]
Start the ResourceManager
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
