This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e2a7008 YARN-10585. Create a class which can convert from legacy
mapping rule format to the new JSON format. Contributed by Gergely Pollak
e2a7008 is described below
commit e2a7008d50d7cf2ec031c98a2b1da8075c48e0ec
Author: Szilard Nemeth <[email protected]>
AuthorDate: Tue Jan 26 18:31:39 2021 +0100
YARN-10585. Create a class which can convert from legacy mapping rule
format to the new JSON format. Contributed by Gergely Pollak
---
.../resourcemanager/placement/MappingRule.java | 6 +-
.../placement/MappingRuleActions.java | 5 +-
.../converter/LegacyMappingRuleToJson.java | 405 +++++++++++++++++++++
.../placement/TestCSMappingPlacementRule.java | 2 +-
.../placement/TestMappingRuleActions.java | 5 +-
.../converter/TestLegacyMappingRuleToJson.java | 240 ++++++++++++
6 files changed, 657 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
index e61ad95..9d67d78 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
@@ -106,7 +106,11 @@ public class MappingRule {
switch (type) {
case USER_MAPPING:
- matcher = MappingRuleMatchers.createUserMatcher(source);
+ if (source.equals("%user")) {
+ matcher = MappingRuleMatchers.createAllMatcher();
+ } else {
+ matcher = MappingRuleMatchers.createUserMatcher(source);
+ }
break;
case GROUP_MAPPING:
matcher = MappingRuleMatchers.createUserGroupMatcher(source);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleActions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleActions.java
index 13cdbe8..35d7276 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleActions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleActions.java
@@ -96,8 +96,9 @@ public final class MappingRuleActions {
@Override
public String toString() {
return "PlaceToQueueAction{" +
- "queueName='" + queuePattern + '\'' +
- '}';
+ "queueName='" + queuePattern + "'," +
+ "allowCreate=" + allowCreate +
+ "}";
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/converter/LegacyMappingRuleToJson.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/converter/LegacyMappingRuleToJson.java
new file mode 100644
index 0000000..113b08b
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/converter/LegacyMappingRuleToJson.java
@@ -0,0 +1,405 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.converter;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.hadoop.util.StringUtils;
+import
org.apache.hadoop.yarn.server.resourcemanager.placement.MappingQueuePath;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+
+public class LegacyMappingRuleToJson {
+ //Legacy rule parse helper constants
+ public static final String RULE_PART_DELIMITER = ":";
+ public static final String PREFIX_USER_MAPPING = "u";
+ public static final String PREFIX_GROUP_MAPPING = "g";
+
+ //Legacy rule matcher variables
+ public static final String MATCHER_APPLICATION = "%application";
+ public static final String MATCHER_USER = "%user";
+
+ //Legacy rule mapping variables, which can be used in target queues
+ public static final String MAPPING_PRIMARY_GROUP = "%primary_group";
+ public static final String MAPPING_SECONDARY_GROUP = "%secondary_group";
+ public static final String MAPPING_USER = MATCHER_USER;
+
+ //JSON Format match all token (actually only used for users)
+ public static final String JSON_MATCH_ALL = "*";
+
+ //Frequently used JSON node names for rule definitions
+ public static final String JSON_NODE_POLICY = "policy";
+ public static final String JSON_NODE_PARENT_QUEUE = "parentQueue";
+ public static final String JSON_NODE_CUSTOM_PLACEMENT = "customPlacement";
+ public static final String JSON_NODE_MATCHES = "matches";
+
+ /**
+ * Our internal object mapper, used to create JSON nodes.
+ */
+ private ObjectMapper objectMapper = new ObjectMapper();
+
+ /**
+ * Collection to store the legacy group mapping rule strings.
+ */
+ private Collection<String> userGroupMappingRules = new ArrayList<>();
+ /**
+ * Collection to store the legacy application name mapping rule strings.
+ */
+ private Collection<String> applicationNameMappingRules = new ArrayList<>();
+
+ /**
+ * This setter method is used to set the raw string format of the legacy
+ * user group mapping rules. This method expect a string formatted just like
+ * in the configuration file of the Capacity Scheduler.
+ * eg. u:bob:root.groups.%primary_group,u:%user:root.default
+ *
+ * @param rules The string containing ALL the UserGroup mapping rules in
+ * legacy format
+ * @return This object for daisy chain support
+ */
+ public LegacyMappingRuleToJson setUserGroupMappingRules(String rules) {
+ setUserGroupMappingRules(StringUtils.getTrimmedStringCollection(rules));
+ return this;
+ }
+
+ /**
+ * This setter method is used to set the the user group mapping rules as a
+ * string collection, where each entry is one rule.
+ *
+ * @param rules One rule per entry
+ * @return This object for daisy chain support
+ */
+ public LegacyMappingRuleToJson setUserGroupMappingRules(
+ Collection<String> rules) {
+ if (rules != null) {
+ userGroupMappingRules = rules;
+ } else {
+ userGroupMappingRules = new ArrayList<>();
+ }
+ return this;
+ }
+
+ /**
+ * This setter method is used to set the raw string format of the legacy
+ * application name mapping rules. This method expect a string formatted
+ * just like in the configuration file of the Capacity Scheduler.
+ * eg. mapreduce:root.apps.%application,%application:root.default
+ *
+ * @param rules The string containing ALL the application name mapping rules
+ * in legacy format
+ * @return This object for daisy chain support
+ */
+ public LegacyMappingRuleToJson setAppNameMappingRules(String rules) {
+ setAppNameMappingRules(StringUtils.getTrimmedStringCollection(rules));
+ return this;
+ }
+
+ /**
+ * This setter method is used to set the the application name mapping rules
as
+ * a string collection, where each entry is one rule.
+ *
+ * @param rules One rule per entry
+ * @return This object for daisy chain support
+ */
+ public LegacyMappingRuleToJson setAppNameMappingRules(
+ Collection<String> rules) {
+ if (rules != null) {
+ applicationNameMappingRules = rules;
+ } else {
+ applicationNameMappingRules = new ArrayList<>();
+ }
+
+ return this;
+ }
+
+ /**
+ * This method will do the conversion based on the already set mapping rules.
+ * First the rules to be converted must be set via setAppNameMappingRules and
+ * setUserGroupMappingRules methods.
+ * @return JSON Format of the provided mapping rules, null if no rules are
set
+ */
+ public String convert() {
+ if (userGroupMappingRules == null && applicationNameMappingRules == null) {
+ return null;
+ }
+
+ //creating the basic JSON config structure
+ ObjectNode rootNode = objectMapper.createObjectNode();
+ ArrayNode rulesNode = objectMapper.createArrayNode();
+ rootNode.set("rules", rulesNode);
+
+ //Processing and adding all the user group mapping rules
+ for (String rule : userGroupMappingRules) {
+ rulesNode.add(convertUserGroupMappingRule(rule));
+ }
+
+ //Processing and adding all the application name mapping rules
+ for (String rule : applicationNameMappingRules) {
+ rulesNode.add(convertAppNameMappingRule(rule));
+ }
+
+ //If there are no converted rules we return null
+ if (rulesNode.size() == 0) {
+ return null;
+ }
+
+ try {
+ return objectMapper
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(rootNode);
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ /**
+ * This intermediate helper method is used to process User Group mapping
rules
+ * and invoke the proper mapping rule creation method.
+ * @param rule The legacy format of the single rule to be converted.
+ * @return The ObjectNode which can be added to the rules part of the config.
+ */
+ ObjectNode convertUserGroupMappingRule(String rule) {
+ String[] mapping = splitRule(rule, 3);
+ String ruleType = mapping[0];
+ String ruleMatch = mapping[1];
+ String ruleTarget = mapping[2];
+
+ if (ruleType.equals(PREFIX_USER_MAPPING)) {
+ return createUserMappingRule(ruleMatch, ruleTarget);
+ }
+
+ if (ruleType.equals(PREFIX_GROUP_MAPPING)) {
+ return createGroupMappingRule(ruleMatch, ruleTarget);
+ }
+
+ throw new IllegalArgumentException(
+ "User group mapping rule must start with prefix '" +
+ PREFIX_USER_MAPPING + "' or '" + PREFIX_GROUP_MAPPING + "'");
+ }
+
+ /**
+ * This intermediate helper method is used to process Application name
mapping
+ * rules and invoke the proper mapping rule creation method.
+ * @param rule The legacy format of the single rule to be converted.
+ * @return The ObjectNode which can be added to the rules part of the config.
+ */
+ ObjectNode convertAppNameMappingRule(String rule) {
+ String[] mapping = splitRule(rule, 2);
+ String ruleMatch = mapping[0];
+ String ruleTarget = mapping[1];
+
+ return createApplicationNameMappingRule(ruleMatch, ruleTarget);
+ }
+ /**
+ * Helper method which splits the rules into parts, and checks if it has
+ * exactly the required amount of parts, and none of them is empty!
+ * @param rule The mapping rule to be split
+ * @param expectedParts The number of expected parts
+ * @return The split String[] of the parts
+ * @throws IllegalArgumentException if the number of parts don't match or any
+ * of them is empty.
+ */
+ private String[] splitRule(String rule, int expectedParts) {
+ //Splitting
+ String[] mapping = StringUtils
+ .getTrimmedStringCollection(rule, RULE_PART_DELIMITER)
+ .toArray(new String[] {});
+
+ //Checking for part count
+ if (mapping.length != expectedParts) {
+ throw new IllegalArgumentException("Invalid rule '" + rule +
+ "' expected parts: " + expectedParts +
+ " actual parts: " + mapping.length);
+ }
+
+ //Checking for empty parts
+ for (int i = 0; i < mapping.length; i++) {
+ if (mapping[i].length() == 0) {
+ throw new IllegalArgumentException("Invalid rule '" + rule +
+ "' with empty part, mapping rules must not contain empty parts!");
+ }
+ }
+
+ return mapping;
+ }
+
+ /**
+ * This helper method is to create a default rule node for the converter,
+ * setting fields which are common in all rules.
+ * @param type The type of the rule can be user/group/application
+ * @return The object node with the preset fields
+ */
+ private ObjectNode createDefaultRuleNode(String type) {
+ return objectMapper
+ .createObjectNode()
+ .put("type", type)
+ //All legacy rule fallback to place to default
+ .put("fallbackResult", "placeDefault")
+ //All legacy rules allow creation
+ .put("create", true);
+ }
+
+ /**
+ * This method will create the JSON node for a single User Mapping Rule.
+ * @param match The match part of the rule it can be either an actual user
+ * name or '%user' to match all users
+ * @param target The queue to place to user into, some queue path variables
+ * are supported (%user, %primary_group, %secondary_group).
+ * @return The ObjectNode which represents the rule
+ */
+ private ObjectNode createUserMappingRule(String match, String target) {
+ ObjectNode ruleNode = createDefaultRuleNode("user");
+ MappingQueuePath targetPath = new MappingQueuePath(target);
+
+ //We have a special token in the JSON format to match all user, replacing
+ //matcher
+ if (match.equals(MATCHER_USER)) {
+ match = JSON_MATCH_ALL;
+ }
+ ruleNode.put(JSON_NODE_MATCHES, match);
+
+ switch (targetPath.getLeafName()) {
+ case MAPPING_USER:
+ ruleNode.put(JSON_NODE_POLICY, "user");
+ if (targetPath.hasParent()) {
+ //Parsing parent path, to be able to determine the short name of parent
+ MappingQueuePath targetParentPath =
+ new MappingQueuePath(targetPath.getParent());
+ String parentShortName = targetParentPath.getLeafName();
+
+ if (parentShortName.equals(MAPPING_PRIMARY_GROUP)) {
+ //%primary_group.%user mapping
+ ruleNode.put(JSON_NODE_POLICY, "primaryGroupUser");
+
+ //Yep, this is confusing. The policy primaryGroupUser actually
+ // appends the %primary_group.%user to the parent path, so we need to
+ // remove it from the parent path to avoid duplication.
+ targetPath = new MappingQueuePath(targetParentPath.getParent(),
+ targetPath.getLeafName());
+ } else if (parentShortName.equals(MAPPING_SECONDARY_GROUP)) {
+ //%secondary_group.%user mapping
+ ruleNode.put(JSON_NODE_POLICY, "secondaryGroupUser");
+
+ //Yep, this is confusing. The policy secondaryGroupUser actually
+ // appends the %secondary_group.%user to the parent path, so we need
+ // to remove it from the parent path to avoid duplication.
+ targetPath = new MappingQueuePath(targetParentPath.getParent(),
+ targetPath.getLeafName());
+ }
+
+ //[parent].%user mapping
+ }
+ break;
+ case MAPPING_PRIMARY_GROUP:
+ //[parent].%primary_group mapping
+ ruleNode.put(JSON_NODE_POLICY, "primaryGroup");
+ break;
+ case MAPPING_SECONDARY_GROUP:
+ //[parent].%secondary_group mapping
+ ruleNode.put(JSON_NODE_POLICY, "secondaryGroup");
+ break;
+ default:
+ //static path mapping
+ ruleNode.put(JSON_NODE_POLICY, "custom");
+ ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
+ break;
+ }
+
+ //if the target queue has a parent part, and the rule can have a parent
+ //we add it to the node
+ if (targetPath.hasParent()) {
+ ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
+ }
+
+ return ruleNode;
+ }
+
+ /**
+ * This method will create the JSON node for a single Group Mapping Rule.
+ * @param match The name of the group to match for
+ * @param target The queue to place to user into, some queue path variables
+ * are supported (%user).
+ * @return The ObjectNode which represents the rule
+ */
+ private ObjectNode createGroupMappingRule(String match, String target) {
+ ObjectNode ruleNode = createDefaultRuleNode("group");
+ MappingQueuePath targetPath = new MappingQueuePath(target);
+
+ //we simply used the source match part all valid legacy matchers are valid
+ //matchers for the JSON format as well
+ ruleNode.put(JSON_NODE_MATCHES, match);
+
+ if (targetPath.getLeafName().matches(MATCHER_USER)) {
+ //g:group:[parent].%user mapping
+ ruleNode.put(JSON_NODE_POLICY, "user");
+
+ //if the target queue has a parent part we add it to the node
+ if (targetPath.hasParent()) {
+ ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
+ }
+ } else {
+ //static path mapping
+ ruleNode.put(JSON_NODE_POLICY, "custom");
+ ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
+ }
+
+ return ruleNode;
+ }
+
+
+ /**
+ * This method will create the JSON node for a single Application Name
+ * Mapping Rule.
+ * @param match The name of the application to match for or %application to
+ * match all applications
+ * @param target The queue to place to user into, some queue path variables
+ * are supported (%application).
+ * @return The ObjectNode which represents the rule
+ */
+ private ObjectNode createApplicationNameMappingRule(
+ String match, String target) {
+ ObjectNode ruleNode = createDefaultRuleNode("application");
+ MappingQueuePath targetPath = new MappingQueuePath(target);
+
+ //we simply used the source match part all valid legacy matchers are valid
+ //matchers for the JSON format as well
+ ruleNode.put(JSON_NODE_MATCHES, match);
+
+ if (targetPath.getLeafName().matches(MATCHER_APPLICATION)) {
+ //[parent].%application mapping
+ ruleNode.put(JSON_NODE_POLICY, "applicationName");
+
+ //if the target queue has a parent part we add it to the node
+ if (targetPath.hasParent()) {
+ ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
+ }
+ } else {
+ //static path mapping
+ ruleNode.put(JSON_NODE_POLICY, "custom");
+ ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
+ }
+
+ return ruleNode;
+ }
+}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java
index 6ee7b5df..703d517 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java
@@ -467,7 +467,7 @@ public class TestCSMappingPlacementRule {
assertTrue("Rule's match value should be bob",
ruleStr.contains("value='bob'"));
assertTrue("Rule's action should be place to queue", ruleStr.contains(
- "action=PlaceToQueueAction{queueName='%primary_group'}"));
+ "action=PlaceToQueueAction{queueName='%primary_group'"));
}
@Test
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleActions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleActions.java
index 769d051..4d4daa1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleActions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleActions.java
@@ -166,9 +166,10 @@ public class TestMappingRuleActions {
"%var", "value");
MappingRuleAction reject = new MappingRuleActions.RejectAction();
- assertEquals("PlaceToQueueAction{queueName='queue'}", place.toString());
+ assertEquals("PlaceToQueueAction{queueName='queue',allowCreate=true}",
+ place.toString());
assertEquals("VariableUpdateAction{variableName='%var'" +
- ", variableValue='value'}", varUpdate.toString());
+ ", variableValue='value'}", varUpdate.toString());
assertEquals("RejectAction", reject.toString());
}
}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/converter/TestLegacyMappingRuleToJson.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/converter/TestLegacyMappingRuleToJson.java
new file mode 100644
index 0000000..9a2b97f
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/converter/TestLegacyMappingRuleToJson.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.converter;
+
+import static org.junit.Assert.*;
+
+import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+public class TestLegacyMappingRuleToJson {
+
+ void validateConversion(String legacyUserGroup, String legacyAppName)
+ throws IOException {
+ //Creating a capacity scheduler config, because this way we can run
+ //both the legacy and the JSON rules through the parser engine, and
+ //we can check if we get the same mapping rules
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+
+ //First we configure the capacity scheduler to parse the legacy config
+ conf.set(
+ CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT,
+ CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_LEGACY);
+ conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, legacyUserGroup);
+ conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING_NAME, legacyAppName);
+
+ //These are the legacyRules generated by CS, this can be used as a
reference
+ //we can test the JSON format against these
+ List<MappingRule> legacyRules = conf.getMappingRules();
+
+ //Converting the legacy format to JSON
+ LegacyMappingRuleToJson converter = new LegacyMappingRuleToJson();
+ String json = converter
+ .setUserGroupMappingRules(legacyUserGroup)
+ .setAppNameMappingRules(legacyAppName)
+ .convert();
+
+ //First we configure the capacity scheduler to parse the CONVERTED JSON
+ conf.set(
+ CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT,
+ CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_JSON);
+ conf.set(CapacitySchedulerConfiguration.MAPPING_RULE_JSON, json);
+
+ //These are the rules which are generated from the JSON format
+ List<MappingRule> jsonRules = conf.getMappingRules();
+
+ //Sanity check
+ assertEquals("Number of rules should mach",
+ legacyRules.size(), jsonRules.size());
+
+ //We expect ALL rules to match no matter if it was parsed from legacy
format
+ //or from JSON
+ for (int i = 0; i < legacyRules.size(); i++) {
+ assertEquals(
+ "Rule #" + i + " should match",
+ legacyRules.get(i).toString(),
+ jsonRules.get(i).toString());
+
+ assertEquals(
+ "Rule #" + i + " fallback should match",
+ legacyRules.get(i).getFallback().toString(),
+ jsonRules.get(i).getFallback().toString());
+ }
+
+ }
+
+ @Test
+ public void testApplicationNameMappingConversion() throws IOException {
+ String appMapping = String.join(",",
+ "namedMatch:simple",
+ "namedMatch:root.deep",
+ "namedMatch:%application",
+ "namedMatch:root.deep.%application",
+ "%application:simple",
+ "%application:root.deep",
+ "%application:%application",
+ "%application:root.deep.%application");
+
+ validateConversion("", appMapping);
+ }
+
+ @Test
+ public void testGroupMappingConversion() throws IOException {
+ String groupMapping = String.join(",",
+ "g:testers:simple",
+ "g:developers:root.very.deep",
+ "g:users:%user",
+ "g:testers:root.very.deep.%user");
+
+ validateConversion(groupMapping, "");
+ }
+
+ @Test
+ public void testUserMappingConversion() throws IOException {
+ String groupMapping = String.join(",",
+ "u:alice:alice",
+ "u:beatrix:root.beatrix",
+ "u:claire:%primary_group",
+ "u:donna:root.deep.%primary_group",
+ "u:emily:%secondary_group",
+ "u:felicity:root.deep.%secondary_group",
+ "u:%user:simple",
+ "u:%user:root.deep",
+ "u:%user:%primary_group",
+ "u:%user:%secondary_group",
+ "u:%user:root.deep.%primary_group",
+ "u:%user:root.deep.%secondary_group",
+ "u:%user:%primary_group.%user",
+ "u:%user:root.%primary_group.%user",
+ "u:%user:root.deep.%primary_group.%user",
+ "u:%user:%secondary_group.%user",
+ "u:%user:root.%secondary_group.%user",
+ "u:%user:root.deep.%secondary_group.%user",
+ "u:%user:%user",
+ "u:%user:root.deep.%user");
+
+ validateConversion(groupMapping, "");
+ }
+
+ @Test
+ public void testTotalConversion() throws IOException {
+ String appMapping = String.join(",",
+ "namedMatch:simple",
+ "namedMatch:root.deep",
+ "namedMatch:%application",
+ "namedMatch:root.deep.%application",
+ "%application:simple",
+ "%application:root.deep",
+ "%application:%application",
+ "%application:root.deep.%application");
+
+ String userGroupMapping = String.join(",",
+ "u:alice:alice",
+ "u:beatrix:root.beatrix",
+ "u:claire:%primary_group",
+ "u:donna:root.deep.%primary_group",
+ "u:emily:%secondary_group",
+ "u:felicity:root.deep.%secondary_group",
+ "u:%user:simple",
+ "u:%user:root.deep",
+ "g:testers:simple",
+ "g:developers:root.very.deep",
+ "g:users:%user",
+ "g:testers:root.very.deep.%user",
+ "u:%user:%primary_group",
+ "u:%user:%secondary_group",
+ "u:%user:root.deep.%primary_group",
+ "u:%user:root.deep.%secondary_group",
+ "u:%user:%primary_group.%user",
+ "u:%user:root.%primary_group.%user",
+ "u:%user:root.deep.%primary_group.%user",
+ "u:%user:%secondary_group.%user",
+ "u:%user:root.%secondary_group.%user",
+ "u:%user:root.deep.%secondary_group.%user",
+ "u:%user:%user",
+ "u:%user:root.%user.something",
+ "u:%user:root.deep.%user");
+
+ validateConversion(userGroupMapping, appMapping);
+ }
+
+ @Test
+ public void testErrorHandling() {
+ LegacyMappingRuleToJson converter = new LegacyMappingRuleToJson();
+ //Empty converter should return null
+ assertNull(converter.convert());
+
+ converter
+ .setAppNameMappingRules("")
+ .setUserGroupMappingRules("");
+ //Empty converter should still return null
+ assertNull(converter.convert());
+
+ converter
+ .setAppNameMappingRules((Collection<String>)null)
+ .setUserGroupMappingRules((Collection<String>)null);
+ //Setting nulls should also result in null return.
+ assertNull(converter.convert());
+
+ try {
+ converter
+ .setAppNameMappingRules("%application:")
+ .setUserGroupMappingRules("")
+ .convert();
+ fail("Empty app name mapping part should throw exception");
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ converter
+ .setAppNameMappingRules("%application:sdfsdf:sdfsfd")
+ .setUserGroupMappingRules("")
+ .convert();
+ fail("Incorrect number of app name mapping parts should throw
exception");
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ converter
+ .setAppNameMappingRules("")
+ .setUserGroupMappingRules("u::root.default")
+ .convert();
+ fail("Empty user group mapping part should throw exception");
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ converter
+ .setAppNameMappingRules("")
+ .setUserGroupMappingRules("u:bob")
+ .convert();
+ fail("Incorrect number of user group mapping parts should " +
+ "throw exception");
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ converter
+ .setAppNameMappingRules("")
+ .setUserGroupMappingRules("X:bob:root.bob")
+ .convert();
+ fail("Invalid user group mapping prefix should throw exception");
+ } catch (IllegalArgumentException e) {}
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
