[31/33] hadoop git commit: YARN-5951. Changes to allow CapacityScheduler to use configuration store
YARN-5951. Changes to allow CapacityScheduler to use configuration store Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcbebf5f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcbebf5f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcbebf5f Branch: refs/heads/YARN-5734 Commit: dcbebf5fdf480832668d6b5b43cfedb869973ba8 Parents: 4c66a8d Author: Jonathan HungAuthored: Mon Jan 30 19:03:48 2017 -0800 Committer: Jonathan Hung Committed: Tue Mar 14 16:29:01 2017 -0700 -- .../scheduler/capacity/CapacityScheduler.java | 36 +-- .../CapacitySchedulerConfiguration.java | 10 +++ .../capacity/conf/CSConfigurationProvider.java | 46 ++ .../conf/FileBasedCSConfigurationProvider.java | 67 .../scheduler/capacity/conf/package-info.java | 29 + .../capacity/TestCapacityScheduler.java | 4 +- 6 files changed, 170 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbebf5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index f6e7942..ed70b0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; @@ -104,6 +103,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation; @@ -162,6 +163,7 @@ public class CapacityScheduler extends private int offswitchPerHeartbeatLimit; + private CSConfigurationProvider csConfProvider; @Override public void setConf(Configuration conf) { @@ -285,7 +287,18 @@ public class CapacityScheduler extends IOException { try { writeLock.lock(); - this.conf = loadCapacitySchedulerConfiguration(configuration); + String confProviderStr = configuration.get( + CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER); + if (confProviderStr.equals( + CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) { +this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext); + } else { +throw new IOException("Invalid CS configuration provider: " + +confProviderStr); + } + this.csConfProvider.init(configuration); + this.conf = this.csConfProvider.loadConfiguration(configuration); validateConf(this.conf); this.minimumAllocation = this.conf.getMinimumAllocation(); initMaximumResourceCapability(this.conf.getMaximumAllocation()); @@ -392,7 +405,7 @@ public class CapacityScheduler extends writeLock.lock(); Configuration configuration = new Configuration(newConf); CapacitySchedulerConfiguration oldConf = this.conf; - this.conf = loadCapacitySchedulerConfiguration(configuration); + this.conf =
[31/33] hadoop git commit: YARN-5951. Changes to allow CapacityScheduler to use configuration store
YARN-5951. Changes to allow CapacityScheduler to use configuration store Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37621ccd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37621ccd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37621ccd Branch: refs/heads/YARN-5734 Commit: 37621ccd216b21788d49e863191579d9243c9526 Parents: e96a0b8 Author: Jonathan HungAuthored: Mon Jan 30 19:03:48 2017 -0800 Committer: Jonathan Hung Committed: Thu Mar 9 16:44:53 2017 -0800 -- .../scheduler/capacity/CapacityScheduler.java | 36 +-- .../CapacitySchedulerConfiguration.java | 10 +++ .../capacity/conf/CSConfigurationProvider.java | 46 ++ .../conf/FileBasedCSConfigurationProvider.java | 67 .../scheduler/capacity/conf/package-info.java | 29 + .../capacity/TestCapacityScheduler.java | 4 +- 6 files changed, 170 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37621ccd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index f6e7942..ed70b0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; @@ -104,6 +103,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation; @@ -162,6 +163,7 @@ public class CapacityScheduler extends private int offswitchPerHeartbeatLimit; + private CSConfigurationProvider csConfProvider; @Override public void setConf(Configuration conf) { @@ -285,7 +287,18 @@ public class CapacityScheduler extends IOException { try { writeLock.lock(); - this.conf = loadCapacitySchedulerConfiguration(configuration); + String confProviderStr = configuration.get( + CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER); + if (confProviderStr.equals( + CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) { +this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext); + } else { +throw new IOException("Invalid CS configuration provider: " + +confProviderStr); + } + this.csConfProvider.init(configuration); + this.conf = this.csConfProvider.loadConfiguration(configuration); validateConf(this.conf); this.minimumAllocation = this.conf.getMinimumAllocation(); initMaximumResourceCapability(this.conf.getMaximumAllocation()); @@ -392,7 +405,7 @@ public class CapacityScheduler extends writeLock.lock(); Configuration configuration = new Configuration(newConf); CapacitySchedulerConfiguration oldConf = this.conf; - this.conf = loadCapacitySchedulerConfiguration(configuration); + this.conf =