This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a7d72c5 YARN-10099. FS-CS converter: handle allow-undeclared-pools
and user-as-default-queue properly and fix misc issues. Contributed by Peter
Bacsko
a7d72c5 is described below
commit a7d72c523ae9d23fad5f2fcc4b40610731ce454a
Author: Szilard Nemeth <[email protected]>
AuthorDate: Thu Jan 30 16:03:38 2020 +0100
YARN-10099. FS-CS converter: handle allow-undeclared-pools and
user-as-default-queue properly and fix misc issues. Contributed by Peter Bacsko
---
.../fair/AllocationFileLoaderService.java | 4 +-
.../converter/FSConfigToCSConfigConverter.java | 105 ++++++++++++++++----
.../fair/converter/PreconditionException.java | 4 +
.../fair/converter/QueuePlacementConverter.java | 12 ++-
.../converter/TestFSConfigToCSConfigConverter.java | 110 +++++++++++++++++++++
.../fair/converter/TestFSQueueConverter.java | 14 ++-
.../converter/TestQueuePlacementConverter.java | 2 +-
.../resources/fair-scheduler-noplacementrules.xml | 84 ++++++++++++++++
8 files changed, 306 insertions(+), 29 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index 829188d..ed891ec 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -96,7 +96,7 @@ public class AllocationFileLoaderService extends
AbstractService {
private Thread reloadThread;
private volatile boolean running = true;
- AllocationFileLoaderService(FairScheduler scheduler) {
+ public AllocationFileLoaderService(FairScheduler scheduler) {
this(SystemClock.getInstance(), scheduler);
}
@@ -186,7 +186,7 @@ public class AllocationFileLoaderService extends
AbstractService {
* classpath, but loaded like a regular File.
*/
@VisibleForTesting
- Path getAllocationFile(Configuration conf)
+ public Path getAllocationFile(Configuration conf)
throws UnsupportedFileSystemException {
String allocFilePath = conf.get(FairSchedulerConfiguration.ALLOCATION_FILE,
FairSchedulerConfiguration.DEFAULT_ALLOCATION_FILE);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
index eae952d..efc8f0e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
@@ -26,26 +26,37 @@ import java.io.OutputStream;
import java.util.List;
import java.util.Map;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.security.ConfiguredYarnAuthorizer;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileParser;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
import com.google.common.annotations.VisibleForTesting;
@@ -77,7 +88,7 @@ public class FSConfigToCSConfigConverter {
private ConversionOptions conversionOptions;
private boolean drfUsed = false;
- private Configuration yarnSiteConfig;
+ private Configuration convertedYarnSiteConfig;
private Configuration capacitySchedulerConfig;
private FSConfigToCSConfigRuleHandler ruleHandler;
@@ -98,11 +109,11 @@ public class FSConfigToCSConfigConverter {
validateParams(params);
prepareOutputFiles(params.getOutputDirectory(), params.isConsole());
loadConversionRules(params.getConversionRulesConfig());
- Configuration conf = createConfiguration(params);
- handleFairSchedulerConfig(params, conf);
+ Configuration inputYarnSiteConfig = getInputYarnSiteConfig(params);
+ handleFairSchedulerConfig(params, inputYarnSiteConfig);
this.clusterResource = getClusterResource(params);
- convert(conf);
+ convert(inputYarnSiteConfig);
}
private void prepareOutputFiles(String outputDirectory, boolean console)
@@ -162,13 +173,10 @@ public class FSConfigToCSConfigConverter {
}
}
- private Configuration createConfiguration(
+ private Configuration getInputYarnSiteConfig(
FSConfigToCSConfigConverterParams params) {
Configuration conf = new YarnConfiguration();
conf.addResource(new Path(params.getYarnSiteXmlConfig()));
- conf.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
- conf.setBoolean(FairSchedulerConfiguration.NO_TERMINAL_RULE_CHECK,
- conversionOptions.isNoRuleTerminalCheck());
return conf;
}
@@ -199,7 +207,7 @@ public class FSConfigToCSConfigConverter {
}
@VisibleForTesting
- void convert(Configuration conf) throws Exception {
+ void convert(Configuration inputYarnSiteConfig) throws Exception {
System.out.println(WARNING_TEXT);
// initialize Fair Scheduler
@@ -207,9 +215,20 @@ public class FSConfigToCSConfigConverter {
PlacementManager placementManager = new PlacementManager();
ctx.setQueuePlacementManager(placementManager);
+ // Prepare a separate config for the FS instance
+ // to force the use of ConfiguredYarnAuthorizer, otherwise
+ // it might use that of Ranger
+ Configuration fsConfig = new Configuration(inputYarnSiteConfig);
+ fsConfig.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
+ fsConfig.setBoolean(FairSchedulerConfiguration.NO_TERMINAL_RULE_CHECK,
+ conversionOptions.isNoRuleTerminalCheck());
+ fsConfig.setClass(YarnConfiguration.YARN_AUTHORIZATION_PROVIDER,
+ ConfiguredYarnAuthorizer.class, YarnAuthorizationProvider.class);
FairScheduler fs = new FairScheduler();
fs.setRMContext(ctx);
- fs.init(conf);
+ fs.init(fsConfig);
+ boolean havePlacementPolicies =
+ checkPlacementPoliciesPresent(fs, inputYarnSiteConfig);
drfUsed = isDrfUsed(fs);
@@ -217,13 +236,13 @@ public class FSConfigToCSConfigConverter {
queueMaxAppsDefault = allocConf.getQueueMaxAppsDefault();
queueMaxAMShareDefault = allocConf.getQueueMaxAMShareDefault();
- yarnSiteConfig = new Configuration(false);
+ convertedYarnSiteConfig = new Configuration(false);
capacitySchedulerConfig = new Configuration(false);
checkUserMaxApps(allocConf);
checkUserMaxAppsDefault(allocConf);
- convertYarnSiteXml(conf);
+ convertYarnSiteXml(inputYarnSiteConfig, havePlacementPolicies);
convertCapacitySchedulerXml(fs);
if (consoleMode) {
@@ -235,7 +254,7 @@ public class FSConfigToCSConfigConverter {
System.out.println();
System.out.println("======= " + YARN_SITE_XML + " =======");
}
- yarnSiteConfig.writeXml(yarnSiteOutputStream);
+ convertedYarnSiteConfig.writeXml(yarnSiteOutputStream);
}
@VisibleForTesting
@@ -248,17 +267,23 @@ public class FSConfigToCSConfigConverter {
this.capacitySchedulerOutputStream = out;
}
- private void convertYarnSiteXml(Configuration conf) {
+ private void convertYarnSiteXml(Configuration inputYarnSiteConfig,
+ boolean havePlacementPolicies) {
FSYarnSiteConverter siteConverter =
new FSYarnSiteConverter();
- siteConverter.convertSiteProperties(conf, yarnSiteConfig, drfUsed);
-
- autoCreateChildQueues = siteConverter.isAutoCreateChildQueues();
+ siteConverter.convertSiteProperties(inputYarnSiteConfig,
+ convertedYarnSiteConfig, drfUsed);
+
+ // See docs: "allow-undeclared-pools" and "user-as-default-queue" are
+ // ignored if we have placement rules
+ autoCreateChildQueues =
+ !havePlacementPolicies && siteConverter.isAutoCreateChildQueues();
+ userAsDefaultQueue =
+ !havePlacementPolicies && siteConverter.isUserAsDefaultQueue();
preemptionEnabled = siteConverter.isPreemptionEnabled();
sizeBasedWeight = siteConverter.isSizeBasedWeight();
- userAsDefaultQueue = siteConverter.isUserAsDefaultQueue();
- checkReservationSystem(conf);
+ checkReservationSystem(inputYarnSiteConfig);
}
private void convertCapacitySchedulerXml(FairScheduler fs) {
@@ -402,6 +427,46 @@ public class FSConfigToCSConfigConverter {
@VisibleForTesting
Configuration getYarnSiteConfig() {
- return yarnSiteConfig;
+ return convertedYarnSiteConfig;
+ }
+
+ /*
+ * Determines whether <queuePlacementPolicy> is present
+ * in the allocation file or not.
+ *
+ * Note that placementManager.getPlacementRules.size()
+ * doesn't work - by default, "allow-undeclared-pools" and
+ * "user-as-default-queue" are translated to policies internally
+ * inside QueuePlacementPolicy.fromConfiguration().
+ *
+ */
+ private boolean checkPlacementPoliciesPresent(FairScheduler scheduler,
+ Configuration inputYarnSiteConfig)
+ throws RuntimeException {
+
+ try (AllocationFileLoaderService loader =
+ new AllocationFileLoaderService(scheduler)){
+
+ Path allocFilePath = loader.getAllocationFile(inputYarnSiteConfig);
+ FileSystem fs = allocFilePath.getFileSystem(inputYarnSiteConfig);
+
+ DocumentBuilderFactory docBuilderFactory =
+ DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
+ Document doc = builder.parse(fs.open(allocFilePath));
+ Element root = doc.getDocumentElement();
+
+ NodeList elements = root.getChildNodes();
+
+ AllocationFileParser allocationFileParser =
+ new AllocationFileParser(elements);
+ allocationFileParser.parse();
+ docBuilderFactory.setIgnoringComments(true);
+
+ return
+ allocationFileParser.getQueuePlacementPolicy().isPresent();
+ } catch (Exception e) {
+ throw new PreconditionException("Unable to parse allocation file", e);
+ }
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/PreconditionException.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/PreconditionException.java
index c372513..80aae9a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/PreconditionException.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/PreconditionException.java
@@ -30,6 +30,10 @@ public class PreconditionException extends RuntimeException {
super(message);
}
+ public PreconditionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
public PreconditionException(String message, MissingArgumentException ex) {
super(message, ex);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
index 8d3182f..c47dd76 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
@@ -34,6 +34,8 @@ class QueuePlacementConverter {
private static final String PRIMARY_GROUP = "%primary_group";
private static final String SECONDARY_GROUP = "%secondary_group";
+ private static final String RULE_SEPARATOR = ",";
+
Map<String, String> convertPlacementPolicy(PlacementManager placementManager,
FSConfigToCSConfigRuleHandler ruleHandler, boolean userAsDefaultQueue) {
StringBuilder mapping = new StringBuilder();
@@ -59,7 +61,7 @@ class QueuePlacementConverter {
} else {
if (!userAsDefaultQueue) {
if (mapping.length() > 0) {
- mapping.append(";");
+ mapping.append(RULE_SEPARATOR);
}
mapping.append("u:" + USER + ":" + USER);
}
@@ -72,18 +74,18 @@ class QueuePlacementConverter {
"yarn.scheduler.capacity.queue-mappings-override.enable", "false");
} else if (rule instanceof PrimaryGroupPlacementRule) {
if (mapping.length() > 0) {
- mapping.append(";");
+ mapping.append(RULE_SEPARATOR);
}
mapping.append("u:" + USER + ":" + PRIMARY_GROUP);
} else if (rule instanceof DefaultPlacementRule) {
DefaultPlacementRule defaultRule = (DefaultPlacementRule) rule;
if (mapping.length() > 0) {
- mapping.append(";");
+ mapping.append(RULE_SEPARATOR);
}
mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName);
} else if (rule instanceof SecondaryGroupExistingPlacementRule) {
if (mapping.length() > 0) {
- mapping.append(";");
+ mapping.append(RULE_SEPARATOR);
}
mapping.append("u:" + USER + ":" + SECONDARY_GROUP);
} else if (!(rule instanceof RejectPlacementRule)) {
@@ -103,7 +105,7 @@ class QueuePlacementConverter {
UserPlacementRule userRule) {
PlacementRule pr = userRule.getParentRule();
if (mapping.length() > 0) {
- mapping.append(";");
+ mapping.append(RULE_SEPARATOR);
}
if (pr instanceof PrimaryGroupPlacementRule) {
mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
index dc51db9..c229d94 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
@@ -27,6 +27,7 @@ import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.conve
import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS;
import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -74,6 +75,9 @@ public class TestFSConfigToCSConfigConverter {
prepareFileName("fair-scheduler-onlyfairpolicy.xml");
private static final String FS_MIXED_POLICY_XML =
prepareFileName("fair-scheduler-orderingpolicy-mixed.xml");
+ private static final String FS_NO_PLACEMENT_RULES_XML =
+ prepareFileName("fair-scheduler-noplacementrules.xml");
+
@Mock
private FSConfigToCSConfigRuleHandler ruleHandler;
@@ -504,6 +508,112 @@ public class TestFSConfigToCSConfigConverter {
CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null));
}
+ @SuppressWarnings("checkstyle:linelength")
+ public void testUserAsDefaultQueueWithPlacementRules() throws Exception {
+ config = new Configuration(false);
+ config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
+ config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ FAIR_SCHEDULER_XML);
+
+ converter.convert(config);
+
+ Configuration convertedConf = getConvertedCSConfig();
+
+ String expectedMappingRules =
+
"u:%user:root.admins.devs.%user,u:%user:root.users.%user,u:%user:root.default";
+ String mappingRules =
+ convertedConf.get(CapacitySchedulerConfiguration.QUEUE_MAPPING);
+ assertEquals("Mapping rules", expectedMappingRules, mappingRules);
+ }
+
+ @Test
+ public void testUserAsDefaultQueueTrueWithoutPlacementRules()
+ throws Exception {
+ testUserAsDefaultQueueWithoutPlacementRules(true);
+ }
+
+ @Test
+ public void testUserAsDefaultQueueFalseWithoutPlacementRules()
+ throws Exception {
+ testUserAsDefaultQueueWithoutPlacementRules(false);
+ }
+
+ private void testUserAsDefaultQueueWithoutPlacementRules(boolean
+ userAsDefaultQueue) throws Exception {
+ config = new Configuration(false);
+ config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
+ config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ FS_NO_PLACEMENT_RULES_XML);
+ config.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
+ userAsDefaultQueue);
+
+ converter.convert(config);
+
+ Configuration convertedConf = getConvertedCSConfig();
+ String mappingRules =
+ convertedConf.get(CapacitySchedulerConfiguration.QUEUE_MAPPING);
+
+ if (userAsDefaultQueue) {
+ assertEquals("Mapping rules", "u:%user:%user", mappingRules);
+ } else {
+ assertEquals("Mapping rules", "u:%user:root.default", mappingRules);
+ }
+ }
+
+ @Test
+ public void testAutoCreateChildQueuesWithPlacementRules() throws Exception {
+ config = new Configuration(false);
+ config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
+ config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ FAIR_SCHEDULER_XML);
+ config.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
+ true);
+
+ converter.convert(config);
+
+ Configuration convertedConf = getConvertedCSConfig();
+ String property =
+ "yarn.scheduler.capacity.root.auto-create-child-queue.enabled";
+ assertNull("Auto-create queue shouldn't be set",
+ convertedConf.get(property));
+ }
+
+ @Test
+ public void testAutoCreateChildQueuesTrueWithoutPlacementRules()
+ throws Exception {
+ testAutoCreateChildQueuesWithoutPlacementRules(true);
+ }
+
+ @Test
+ public void testAutoCreateChildQueuesFalseWithoutPlacementRules()
+ throws Exception {
+ testAutoCreateChildQueuesWithoutPlacementRules(false);
+ }
+
+ private void testAutoCreateChildQueuesWithoutPlacementRules(
+ boolean allowUndeclaredPools) throws Exception {
+ config = new Configuration(false);
+ config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
+ config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ FS_NO_PLACEMENT_RULES_XML);
+ config.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
+ allowUndeclaredPools);
+
+ converter.convert(config);
+
+ Configuration convertedConf = getConvertedCSConfig();
+ String property =
+ "yarn.scheduler.capacity.root.auto-create-child-queue.enabled";
+
+ if (allowUndeclaredPools) {
+ assertEquals("Auto-create queue wasn't enabled", true,
+ convertedConf.getBoolean(property, false));
+ } else {
+ assertNull("Auto-create queue shouldn't be set",
+ convertedConf.get(property));
+ }
+ }
+
private Configuration getConvertedCSConfig() {
ByteArrayInputStream input =
new ByteArrayInputStream(csConfigOut.toByteArray());
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
index 2221364..fe4faab 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
@@ -334,7 +334,6 @@ public class TestFSQueueConverter {
@Test
public void testQueueAutoCreateChildQueue() {
- config.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, true);
converter = builder
.withCapacitySchedulerConfig(csConfig)
.withAutoCreateChildQueues(true)
@@ -347,6 +346,19 @@ public class TestFSQueueConverter {
}
@Test
+ public void testQueueWithNoAutoCreateChildQueue() {
+ converter = builder
+ .withCapacitySchedulerConfig(csConfig)
+ .withAutoCreateChildQueues(false)
+ .build();
+
+ converter.convertQueueHierarchy(rootQueue);
+
+ assertNoValueForQueues(ALL_QUEUES, ".auto-create-child-queue.enabled",
+ csConfig);
+ }
+
+ @Test
public void testQueueSizeBasedWeightEnabled() {
converter = builder.withSizeBasedWeight(true).build();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
index bd8ca5e..289d35b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
@@ -224,7 +224,7 @@ public class TestQueuePlacementConverter {
Map<String, String> properties = convert(false);
verifyMapping(properties,
- "u:%user:%user;u:%user:%primary_group;u:%user:%secondary_group");
+ "u:%user:%user,u:%user:%primary_group,u:%user:%secondary_group");
verifyZeroInteractions(ruleHandler);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-noplacementrules.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-noplacementrules.xml
new file mode 100644
index 0000000..ac1383b
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-noplacementrules.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<allocations>
+ <queue name="root">
+ <weight>1.0</weight>
+ <schedulingPolicy>drf</schedulingPolicy>
+ <aclSubmitApps>alice,bob,joe,john hadoop_users</aclSubmitApps>
+ <aclAdministerApps>alice,bob,joe,john hadoop_users</aclAdministerApps>
+ <queue name="default">
+ <weight>1.0</weight>
+ <schedulingPolicy>drf</schedulingPolicy>
+ </queue>
+ <queue name="users" type="parent">
+ <weight>1.0</weight>
+ <schedulingPolicy>drf</schedulingPolicy>
+ <queue name="john">
+ <weight>1.0</weight>
+ <schedulingPolicy>drf</schedulingPolicy>
+ <aclSubmitApps>john </aclSubmitApps>
+ <aclAdministerApps>john </aclAdministerApps>
+
<maxContainerAllocation>vcores=2,memory-mb=8192</maxContainerAllocation>
+ </queue>
+ <queue name="joe">
+ <maxResources>memory-mb=50.0%, vcores=50.0%</maxResources>
+ <weight>3.0</weight>
+ <allowPreemptionFrom>false</allowPreemptionFrom>
+ <schedulingPolicy>drf</schedulingPolicy>
+ <aclSubmitApps>joe </aclSubmitApps>
+ <aclAdministerApps>joe </aclAdministerApps>
+ </queue>
+ </queue>
+ <queue name="admins" type="parent">
+ <maxChildResources>memory-mb=8192, vcores=1</maxChildResources>
+ <weight>1.0</weight>
+ <schedulingPolicy>drf</schedulingPolicy>
+
<maxContainerAllocation>vcores=3,memory-mb=4096</maxContainerAllocation>
+ <queue name="alice">
+ <maxResources>memory-mb=16384, vcores=4</maxResources>
+ <maxRunningApps>2</maxRunningApps>
+ <weight>3.0</weight>
+ <allowPreemptionFrom>false</allowPreemptionFrom>
+ <schedulingPolicy>drf</schedulingPolicy>
+ <aclSubmitApps>alice </aclSubmitApps>
+ <aclAdministerApps>alice </aclAdministerApps>
+ <maxAMShare>0.15</maxAMShare>
+ <reservation>memory-mb=16384, vcores=4</reservation>
+ </queue>
+ <queue name="bob">
+ <maxResources>memory-mb=8192, vcores=2</maxResources>
+ <weight>1.0</weight>
+ <schedulingPolicy>drf</schedulingPolicy>
+ <aclSubmitApps>bob </aclSubmitApps>
+ <aclAdministerApps>bob </aclAdministerApps>
+ <maxAMShare>-1.0</maxAMShare>
+ </queue>
+ </queue>
+ </queue>
+ <user name="alice">
+ <maxRunningApps>30</maxRunningApps>
+ </user>
+ <userMaxAppsDefault>10</userMaxAppsDefault>
+ <defaultFairSharePreemptionTimeout>23</defaultFairSharePreemptionTimeout>
+ <defaultMinSharePreemptionTimeout>24</defaultMinSharePreemptionTimeout>
+
<defaultFairSharePreemptionThreshold>0.12</defaultFairSharePreemptionThreshold>
+ <queueMaxAppsDefault>15</queueMaxAppsDefault>
+ <defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>
+ <queueMaxAMShareDefault>0.16</queueMaxAMShareDefault>
+</allocations>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
