This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new f76a2a7 YARN-10654. Dots '.' in CSMappingRule path variables should
be replaced. Contributed by Peter Bacsko
f76a2a7 is described below
commit f76a2a7606760eaf9967dae7ab769d22b11b4447
Author: Szilard Nemeth <[email protected]>
AuthorDate: Fri Apr 23 16:07:58 2021 +0200
YARN-10654. Dots '.' in CSMappingRule path variables should be replaced.
Contributed by Peter Bacsko
---
.../placement/CSMappingPlacementRule.java | 22 +++++--
.../csmappingrule/TestCSMappingPlacementRule.java | 67 +++++++++++++++++++---
2 files changed, 77 insertions(+), 12 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
index 894bc82..9648769 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -42,8 +42,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
-
/**
* This class is responsible for making application submissions to queue
* assignments, based on the configured ruleset. This class supports all
@@ -55,6 +53,8 @@ import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
public class CSMappingPlacementRule extends PlacementRule {
private static final Logger LOG = LoggerFactory
.getLogger(CSMappingPlacementRule.class);
+ private static final String DOT = ".";
+ private static final String DOT_REPLACEMENT = "_dot_";
private CapacitySchedulerQueueManager queueManager;
private List<MappingRule> mappingRules;
@@ -194,12 +194,13 @@ public class CSMappingPlacementRule extends PlacementRule
{
return;
}
Iterator<String> it = groupsSet.iterator();
- String primaryGroup = it.next();
+ String primaryGroup = cleanName(it.next());
ArrayList<String> secondaryGroupList = new ArrayList<>();
while (it.hasNext()) {
- secondaryGroupList.add(it.next());
+ String groupName = cleanName(it.next());
+ secondaryGroupList.add(groupName);
}
if (secondaryGroupList.size() == 0) {
@@ -226,7 +227,7 @@ public class CSMappingPlacementRule extends PlacementRule {
ApplicationSubmissionContext asc, String user) {
VariableContext vctx = new VariableContext();
- vctx.put("%user", user);
+ vctx.put("%user", cleanName(user));
//If the specified matches the default it means NO queue have been
specified
//as per ClientRMService#submitApplication which sets the queue to default
//when no queue is provided.
@@ -522,4 +523,15 @@ public class CSMappingPlacementRule extends PlacementRule {
" mapping rule. Please see the logs for details");
}
}
+
+ private String cleanName(String name) {
+ if (name.contains(DOT)) {
+ String converted = name.replaceAll("\\.", DOT_REPLACEMENT);
+ LOG.warn("Name {} is converted to {} when it is used as a queue name.",
+ name, converted);
+ return converted;
+ } else {
+ return name;
+ }
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
index f6bb2a1..0cf1059 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
@@ -63,13 +63,15 @@ public class TestCSMappingPlacementRule {
@Rule
public TemporaryFolder folder = new TemporaryFolder();
- private Map<String, Set<String>> userGroups = ImmutableMap.of(
- "alice", ImmutableSet.of("p_alice", "unique", "user"),
- "bob", ImmutableSet.of("p_bob", "user", "developer"),
- "charlie", ImmutableSet.of("p_charlie", "user", "tester"),
- "dave", ImmutableSet.of("user"),
- "emily", ImmutableSet.of("user", "tester", "developer")
- );
+ private Map<String, Set<String>> userGroups =
+ ImmutableMap.<String, Set<String>>builder()
+ .put("alice", ImmutableSet.of("p_alice", "unique", "user"))
+ .put("bob", ImmutableSet.of("p_bob", "user", "developer"))
+ .put("charlie", ImmutableSet.of("p_charlie", "user", "tester"))
+ .put("dave", ImmutableSet.of("user"))
+ .put("emily", ImmutableSet.of("user", "tester", "developer"))
+ .put("test.user", ImmutableSet.of("main.grp", "sec.test.grp"))
+ .build();
private void createQueueHierarchy(CapacitySchedulerQueueManager
queueManager) {
MockQueueHierarchyBuilder.create()
@@ -79,6 +81,9 @@ public class TestCSMappingPlacementRule {
.withManagedParentQueue("root.man")
.withQueue("root.user.alice")
.withQueue("root.user.bob")
+ .withQueue("root.user.test_dot_user")
+ .withQueue("root.groups.main_dot_grp")
+ .withQueue("root.groups.sec_dot_test_dot_grp")
.withQueue("root.secondaryTests.unique")
.withQueue("root.secondaryTests.user")
.withQueue("root.ambiguous.user.charlie")
@@ -718,4 +723,52 @@ public class TestCSMappingPlacementRule {
assertConfigTestResult(rules);
}
+
+ @Test
+ public void testUserNameCleanup() throws IOException {
+ ArrayList<MappingRule> rules = new ArrayList<>();
+ rules.add(
+ new MappingRule(
+ MappingRuleMatchers.createAllMatcher(),
+ (new MappingRuleActions.PlaceToQueueAction("%user", true))
+ .setFallbackReject()));
+
+ CSMappingPlacementRule engine = setupEngine(true, rules);
+ ApplicationSubmissionContext app = createApp("app");
+ assertPlace(
+ "test.user should be placed to root.users.test_dot_user",
+ engine, app, "test.user", "root.user.test_dot_user");
+ }
+
+ @Test
+ public void testPrimaryGroupNameCleanup() throws IOException {
+ ArrayList<MappingRule> rules = new ArrayList<>();
+ rules.add(
+ new MappingRule(
+ MappingRuleMatchers.createAllMatcher(),
+ (new MappingRuleActions.PlaceToQueueAction("%primary_group", true))
+ .setFallbackReject()));
+
+ CSMappingPlacementRule engine = setupEngine(true, rules);
+ ApplicationSubmissionContext app = createApp("app");
+ assertPlace(
+ "Application should have been placed to root.groups.main_dot_grp",
+ engine, app, "test.user", "root.groups.main_dot_grp");
+ }
+
+ @Test
+ public void testSecondaryGroupNameCleanup() throws IOException {
+ ArrayList<MappingRule> rules = new ArrayList<>();
+ rules.add(
+ new MappingRule(
+ MappingRuleMatchers.createAllMatcher(),
+ (new MappingRuleActions.PlaceToQueueAction("%secondary_group",
true))
+ .setFallbackReject()));
+
+ CSMappingPlacementRule engine = setupEngine(true, rules);
+ ApplicationSubmissionContext app = createApp("app");
+ assertPlace(
+ "Application should have been placed to
root.groups.sec_dot_test_dot_grp",
+ engine, app, "test.user", "root.groups.sec_dot_test_dot_grp");
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
