[33/33] hadoop git commit: YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store
YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25d2028b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25d2028b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25d2028b Branch: refs/heads/YARN-5734 Commit: 25d2028be4ff2223489e149d63de66895f8449ec Parents: 874b9b8 Author: Jonathan HungAuthored: Wed Mar 1 16:03:01 2017 -0800 Committer: Jonathan Hung Committed: Tue Mar 14 16:29:32 2017 -0700 -- .../hadoop/yarn/conf/YarnConfiguration.java | 6 ++ .../src/main/resources/yarn-default.xml | 12 +++ .../scheduler/MutableConfigurationProvider.java | 35 .../scheduler/capacity/CapacityScheduler.java | 14 ++- .../CapacitySchedulerConfiguration.java | 3 + .../capacity/conf/CSConfigurationProvider.java | 3 +- .../conf/MutableCSConfigurationProvider.java| 94 .../conf/YarnConfigurationStoreFactory.java | 46 ++ .../TestMutableCSConfigurationProvider.java | 83 + 9 files changed, 291 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/25d2028b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b366855..91728a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -602,6 +602,12 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = "org.apache.hadoop.yarn.LocalConfigurationProvider"; + public static final String SCHEDULER_CONFIGURATION_STORE_CLASS = + YARN_PREFIX + "scheduler.configuration.store.class"; + public static final String MEMORY_CONFIGURATION_STORE = "memory"; + public static final String DEFAULT_CONFIGURATION_STORE = + MEMORY_CONFIGURATION_STORE; + public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX + "authorization-provider"; private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = http://git-wip-us.apache.org/repos/asf/hadoop/blob/25d2028b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 727e2c9..9a3fa8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3079,4 +3079,16 @@ false + + + The type of configuration store to use for storing scheduler + configurations, if using a mutable configuration provider. + Keywords such as "memory" map to certain configuration store + implementations. If keyword is not found, try to load this + value as a class. + +yarn.scheduler.configuration.store.class +memory + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/25d2028b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java new file mode 100644 index 000..da30a2b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *
[33/33] hadoop git commit: YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store
YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d219f00 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d219f00 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d219f00 Branch: refs/heads/YARN-5734 Commit: 1d219f00ecdb074257bef21361cc3c33aa063edd Parents: 7b3d27c Author: Jonathan HungAuthored: Wed Mar 1 16:03:01 2017 -0800 Committer: Jonathan Hung Committed: Thu Mar 9 16:45:04 2017 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 6 ++ .../src/main/resources/yarn-default.xml | 12 +++ .../scheduler/MutableConfigurationProvider.java | 35 .../scheduler/capacity/CapacityScheduler.java | 14 ++- .../CapacitySchedulerConfiguration.java | 3 + .../capacity/conf/CSConfigurationProvider.java | 3 +- .../conf/MutableCSConfigurationProvider.java| 94 .../conf/YarnConfigurationStoreFactory.java | 46 ++ .../TestMutableCSConfigurationProvider.java | 83 + 9 files changed, 291 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d219f00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b366855..91728a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -602,6 +602,12 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = "org.apache.hadoop.yarn.LocalConfigurationProvider"; + public static final String SCHEDULER_CONFIGURATION_STORE_CLASS = + YARN_PREFIX + "scheduler.configuration.store.class"; + public static final String MEMORY_CONFIGURATION_STORE = "memory"; + public static final String DEFAULT_CONFIGURATION_STORE = + MEMORY_CONFIGURATION_STORE; + public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX + "authorization-provider"; private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d219f00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 645a342..2918bd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3068,4 +3068,16 @@ 64 + + + The type of configuration store to use for storing scheduler + configurations, if using a mutable configuration provider. + Keywords such as "memory" map to certain configuration store + implementations. If keyword is not found, try to load this + value as a class. + +yarn.scheduler.configuration.store.class +memory + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d219f00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java new file mode 100644 index 000..da30a2b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + *