This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 971f1b8 YARN-10872. Replace getPropsWithPrefix calls in
AutoCreatedQueueTemplate (#3396)
971f1b8 is described below
commit 971f1b8b0a37b4d3eafdecf705afa4f684253e26
Author: Benjamin Teke <brumi1...@users.noreply.github.com>
AuthorDate: Fri Sep 10 17:32:42 2021 +0200
YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate
(#3396)
Co-authored-by: Benjamin Teke <bt...@cloudera.com>
---
.../capacity/AbstractManagedParentQueue.java | 37 ++++------------------
.../capacity/AutoCreatedQueueTemplate.java | 24 ++++++++------
.../capacity/CapacitySchedulerConfiguration.java | 5 +--
.../capacity/ConfigurationProperties.java | 12 ++++++-
.../capacity/TestConfigurationProperties.java | 11 +++++++
5 files changed, 46 insertions(+), 43 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
index a9e82a6..9c16de0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
@@ -28,11 +28,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
/**
* A container class for automatically created child leaf queues.
@@ -173,44 +170,22 @@ public abstract class AbstractManagedParentQueue extends
ParentQueue {
return queueManagementPolicy;
}
- protected SortedMap<String, String> getConfigurationsWithPrefix
- (SortedMap<String, String> sortedConfigs, String prefix) {
- return sortedConfigs.subMap( prefix, prefix + Character.MAX_VALUE );
- }
-
- protected SortedMap<String, String> sortCSConfigurations() {
- SortedMap<String, String> sortedConfigs = new TreeMap(
- new Comparator<String>() {
- public int compare(String s1, String s2) {
- return s1.compareToIgnoreCase(s2);
- }
-
- });
-
- for (final Iterator<Map.Entry<String, String>> iterator =
- csContext.getConfiguration().iterator(); iterator.hasNext(); ) {
- final Map.Entry<String, String> confKeyValuePair = iterator.next();
- sortedConfigs.put(confKeyValuePair.getKey(),
confKeyValuePair.getValue());
- }
- return sortedConfigs;
- }
-
protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String
configPrefix) {
CapacitySchedulerConfiguration leafQueueConfigs = new
CapacitySchedulerConfiguration(new Configuration(false), false);
- String prefix = YarnConfiguration.RESOURCE_TYPES + ".";
Map<String, String> rtProps = csContext
- .getConfiguration().getPropsWithPrefix(prefix);
+ .getConfiguration().getConfigurationProperties()
+ .getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true);
for (Map.Entry<String, String> entry : rtProps.entrySet()) {
- leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue());
+ leafQueueConfigs.set(entry.getKey(), entry.getValue());
}
- SortedMap<String, String> sortedConfigs = sortCSConfigurations();
- SortedMap<String, String> templateConfigs = getConfigurationsWithPrefix
- (sortedConfigs, configPrefix);
+ Map<String, String> templateConfigs = csContext
+ .getConfiguration().getConfigurationProperties()
+ .getPropertiesWithPrefix(configPrefix, true);
for (final Iterator<Map.Entry<String, String>> iterator =
templateConfigs.entrySet().iterator(); iterator.hasNext(); ) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
index cb0f789..eff8c4e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import org.apache.hadoop.conf.Configuration;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
@@ -50,7 +49,7 @@ public class AutoCreatedQueueTemplate {
private final Map<String, String> leafOnlyProperties = new HashMap<>();
private final Map<String, String> parentOnlyProperties = new HashMap<>();
- public AutoCreatedQueueTemplate(Configuration configuration,
+ public AutoCreatedQueueTemplate(CapacitySchedulerConfiguration configuration,
String queuePath) {
setTemplateConfigEntries(configuration, queuePath);
}
@@ -91,7 +90,7 @@ public class AutoCreatedQueueTemplate {
* @param conf configuration to set
* @param childQueuePath child queue path used for prefixing the properties
*/
- public void setTemplateEntriesForChild(Configuration conf,
+ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
String childQueuePath) {
setTemplateEntriesForChild(conf, childQueuePath, false);
}
@@ -105,16 +104,20 @@ public class AutoCreatedQueueTemplate {
* parent specific template properties
* @param childQueuePath child queue path used for prefixing the properties
*/
- public void setTemplateEntriesForChild(Configuration conf,
+ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
String childQueuePath,
boolean isLeaf) {
if (childQueuePath.equals(ROOT)) {
return;
}
+ ConfigurationProperties configurationProperties =
+ conf.getConfigurationProperties();
+
// Get all properties that are explicitly set
- Set<String> alreadySetProps = conf.getPropsWithPrefix(
-
CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet();
+ Set<String> alreadySetProps = configurationProperties
+ .getPropertiesWithPrefix(CapacitySchedulerConfiguration
+ .getQueuePrefix(childQueuePath)).keySet();
// Check template properties only set for leaf or parent queues
Map<String, String> queueTypeSpecificTemplates = parentOnlyProperties;
@@ -151,8 +154,11 @@ public class AutoCreatedQueueTemplate {
* yarn.scheduler.capacity.root.a.auto-queue-creation-v2.template.capacity
* yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity
*/
- private void setTemplateConfigEntries(Configuration configuration,
+ private void setTemplateConfigEntries(CapacitySchedulerConfiguration
configuration,
String queuePath) {
+ ConfigurationProperties configurationProperties =
+ configuration.getConfigurationProperties();
+
List<String> queuePathParts = new ArrayList<>(Arrays.asList(
queuePath.split("\\.")));
@@ -178,8 +184,8 @@ public class AutoCreatedQueueTemplate {
String templateQueuePath = String.join(".", queuePathParts);
// Get all configuration entries with
// yarn.scheduler.capacity.<queuePath> prefix
- Map<String, String> queueProps = configuration.getPropsWithPrefix(
- getQueuePrefix(templateQueuePath));
+ Map<String, String> queueProps = configurationProperties
+ .getPropertiesWithPrefix(getQueuePrefix(templateQueuePath));
// Store template, parent-template and leaf-template properties
for (Map.Entry<String, String> entry : queueProps.entrySet()) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 2bc4077..510559d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -1607,8 +1607,9 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
*/
public Map<String, Set<String>> getConfiguredNodeLabelsByQueue() {
Map<String, Set<String>> labelsByQueue = new HashMap<>();
- Map<String, String> schedulerEntries = getPropsWithPrefix(
- CapacitySchedulerConfiguration.PREFIX);
+ Map<String, String> schedulerEntries =
+ getConfigurationProperties().getPropertiesWithPrefix(
+ CapacitySchedulerConfiguration.PREFIX);
for (Map.Entry<String, String> propertyEntry
: schedulerEntries.entrySet()) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java
index fa05b9f..a24702e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java
@@ -76,7 +76,17 @@ public class ConfigurationProperties {
String prefix, boolean fullyQualifiedKey) {
List<String> propertyPrefixParts = splitPropertyByDelimiter(prefix);
Map<String, String> properties = new HashMap<>();
- String trimPrefix = fullyQualifiedKey ? "" : prefix;
+ String trimPrefix;
+ if (fullyQualifiedKey) {
+ trimPrefix = "";
+ } else {
+ // To support the behaviour where the
+ // CapacitySchedulerConfiguration.getQueuePrefix(String queue) method
+ // returned with the queue prefix with a dot appended to it the last dot
+ // should be removed
+ trimPrefix = prefix.endsWith(CapacitySchedulerConfiguration.DOT) ?
+ prefix.substring(0, prefix.length() - 1) : prefix;
+ }
collectPropertiesRecursively(nodes, properties,
propertyPrefixParts.iterator(), trimPrefix);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java
index 41820da..3f04670 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java
@@ -58,6 +58,17 @@ public class TestConfigurationProperties {
Assert.assertTrue(props.containsKey("4.5"));
Assert.assertEquals("TEST_VALUE_3_2", props.get("4.5"));
+ // Test the scenario where the prefix has a dot appended to it
+ // (see CapacitySchedulerConfiguration.getQueuePrefix(String queue)).
+ // The dot is disregarded.
+ props = configurationProperties
+ .getPropertiesWithPrefix("root.1.2.4.");
+
+ Assert.assertEquals(2, props.size());
+ Assert.assertTrue(props.containsKey(""));
+ Assert.assertEquals("TEST_VALUE_3_1", props.get(""));
+ Assert.assertTrue(props.containsKey("5"));
+ Assert.assertEquals("TEST_VALUE_3_2", props.get("5"));
Map<String, String> propsWithRootPrefix = configurationProperties
.getPropertiesWithPrefix("root");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
