This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 56a5c36 YARN-10373. Create Matchers for CS mapping rules. Contributed
by Gergely Pollak
56a5c36 is described below
commit 56a5c360a101470b56e40a021891b73a26e6db73
Author: Szilard Nemeth <[email protected]>
AuthorDate: Sat Aug 29 21:34:55 2020 +0200
YARN-10373. Create Matchers for CS mapping rules. Contributed by Gergely
Pollak
---
.../placement/MappingRuleMatcher.java | 28 +++
.../placement/MappingRuleMatchers.java | 239 +++++++++++++++++++
.../placement/TestMappingRuleMatchers.java | 255 +++++++++++++++++++++
3 files changed, 522 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatcher.java
new file mode 100644
index 0000000..d2650e9
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatcher.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+public interface MappingRuleMatcher {
+ /**
+ * Returns true if the matcher matches the current context.
+ * @param variables The variable context, which contains all the variables
+ * @return true if this matcher matches to the provided variable set
+ */
+ boolean match(VariableContext variables);
+}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatchers.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatchers.java
new file mode 100644
index 0000000..fdc239f
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatchers.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import java.util.Arrays;
+
+/**
+ * This class contains all the matcher and some helper methods to generate
them.
+ */
+public class MappingRuleMatchers {
+ /**
+ * Utility class, hiding constructor.
+ */
+ private MappingRuleMatchers() {}
+
+ /**
+ * MatchAllMatcher is a matcher which matches everything.
+ */
+ public static class MatchAllMatcher implements MappingRuleMatcher {
+ /**
+ * The match will return true in all cases, to match all submissions.
+ * @param variables The variable context, which contains all the variables
+ * @return true
+ */
+ @Override
+ public boolean match(VariableContext variables) {
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "MatchAllMatcher";
+ }
+ }
+
+ /**
+ * VariableMatcher will check if a provided variable's value matches the
+ * provided value. The provided value might contain variables as well, which
+ * will get evaluated before the comparison.
+ */
+ public static class VariableMatcher implements MappingRuleMatcher {
+ /**
+ * Name of the variable to be checked.
+ */
+ private String variable;
+ /**
+ * The value which should match the variable's value.
+ */
+ private String value;
+
+ VariableMatcher(String variable, String value) {
+ this.variable = variable;
+ this.value = value == null ? "" : value;
+ }
+
+ /**
+ * The method will replace all variables in the value, then compares this
+ * substituted value against the variable's value, if they match we return
+ * true.
+ * If the variable is null we always return false.
+ * @param variables The variable context, which contains all the variables
+ * @return true if the value matches the variable's value, false otherwise
+ */
+ @Override
+ public boolean match(VariableContext variables) {
+ if (variable == null) {
+ return false;
+ }
+
+ String substituted = variables.replaceVariables(value);
+ return substituted.equals(variables.get(variable));
+ }
+
+ @Override
+ public String toString() {
+ return "VariableMatcher{" +
+ "variable='" + variable + '\'' +
+ ", value='" + value + '\'' +
+ '}';
+ }
+ }
+
+ /**
+ * AndMatcher is a basic boolean matcher which takes multiple other
+ * matcher as it's arguments, and on match it checks if all of them are true.
+ */
+ public static class AndMatcher implements MappingRuleMatcher {
+ /**
+ * The list of matchers to be checked during evaluation.
+ */
+ private MappingRuleMatcher[] matchers;
+
+ /**
+ * Constructor.
+ * @param matchers List of matchers to be checked during evaluation
+ */
+ AndMatcher(MappingRuleMatcher...matchers) {
+ this.matchers = matchers;
+ }
+
+ /**
+ * This match method will go through all the provided matchers and call
+ * their match method, if all match we return true.
+ * @param variables The variable context, which contains all the variables
+ * @return true if all matchers match
+ */
+ @Override
+ public boolean match(VariableContext variables) {
+ for (MappingRuleMatcher matcher : matchers) {
+ if (!matcher.match(variables)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "AndMatcher{" +
+ "matchers=" + Arrays.toString(matchers) +
+ '}';
+ }
+ }
+
+ /**
+ * OrMatcher is a basic boolean matcher which takes multiple other
+ * matcher as its arguments, and on match it checks if any of them are true.
+ */
+ public static class OrMatcher implements MappingRuleMatcher {
+ /**
+ * The list of matchers to be checked during evaluation.
+ */
+ private MappingRuleMatcher[] matchers;
+
+ /**
+ * Constructor.
+ * @param matchers List of matchers to be checked during evaluation
+ */
+ OrMatcher(MappingRuleMatcher...matchers) {
+ this.matchers = matchers;
+ }
+
+ /**
+ * This match method will go through all the provided matchers and call
+ * their match method, if any of them match we return true.
+ * @param variables The variable context, which contains all the variables
+ * @return true if any of the matchers match
+ */
+ @Override
+ public boolean match(VariableContext variables) {
+ for (MappingRuleMatcher matcher : matchers) {
+ if (matcher.match(variables)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "OrMatcher{" +
+ "matchers=" + Arrays.toString(matchers) +
+ '}';
+ }
+ }
+
+ /**
+ * Convenience method to create a variable matcher which matches against the
+ * username.
+ * @param userName The username to be matched
+ * @return VariableMatcher with %user as the variable
+ */
+ public static MappingRuleMatcher createUserMatcher(String userName) {
+ return new VariableMatcher("%user", userName);
+ }
+
+ /**
+ * Convenience method to create a variable matcher which matches against the
+ * user's primary group.
+ * @param groupName The groupName to be matched
+ * @return VariableMatcher with %primary_group as the variable
+ */
+ public static MappingRuleMatcher createGroupMatcher(String groupName) {
+ return new VariableMatcher("%primary_group", groupName);
+ }
+
+ /**
+ * Convenience method to create a composite matcher which matches against the
+ * user's user name and the user's primary group. Only matches if both
+ * matches.
+ * @param userName The username to be matched
+ * @param groupName The groupName to be matched
+ * @return AndMatcher with two matchers one for userName and one for
+ * primaryGroup
+ */
+ public static MappingRuleMatcher createUserGroupMatcher(
+ String userName, String groupName) {
+ return new AndMatcher(
+ createUserMatcher(userName),
+ createGroupMatcher(groupName));
+ }
+
+ /**
+ * Convenience method to create a variable matcher which matches against the
+ * submitted application's name.
+ * @param name The name to be matched
+ * @return VariableMatcher with %application as the variable
+ */
+ public static MappingRuleMatcher createApplicationNameMatcher(String name) {
+ return new VariableMatcher("%application", name);
+ }
+
+
+ /**
+ * Convenience method to create a matcher that matches all
+ * @return MatchAllMatcher.
+ */
+ public static MappingRuleMatcher createAllMatcher() {
+ return new MatchAllMatcher();
+ }
+}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleMatchers.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleMatchers.java
new file mode 100644
index 0000000..d384f93
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleMatchers.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestMappingRuleMatchers extends TestCase {
+
+ @Test
+ public void testCatchAll() {
+ VariableContext variables = new VariableContext();
+ MappingRuleMatcher matcher = new MappingRuleMatchers.MatchAllMatcher();
+
+ assertTrue(matcher.match(variables));
+ }
+
+ @Test
+ public void testVariableMatcher() {
+ VariableContext matchingContext = new VariableContext();
+ matchingContext.put("%user", "bob");
+ matchingContext.put("%primary_group", "developers");
+ matchingContext.put("%application", "TurboMR");
+ matchingContext.put("%custom", "Matching string");
+
+ VariableContext mismatchingContext = new VariableContext();
+ mismatchingContext.put("%user", "dave");
+ mismatchingContext.put("%primary_group", "testers");
+ mismatchingContext.put("%application", "Tester APP");
+ mismatchingContext.put("%custom", "Not matching string");
+
+ VariableContext emptyContext = new VariableContext();
+
+ Map<String, MappingRuleMatcher> matchers = new HashMap<>();
+ matchers.put("User matcher", MappingRuleMatchers.createUserMatcher("bob"));
+ matchers.put("Group matcher",
+ MappingRuleMatchers.createGroupMatcher("developers"));
+ matchers.put("Application name matcher",
+ MappingRuleMatchers.createApplicationNameMatcher("TurboMR"));
+ matchers.put("Custom matcher",
+ new MappingRuleMatchers.VariableMatcher("%custom", "Matching string"));
+
+ matchers.forEach((matcherName, matcher) -> {
+ assertTrue(matcherName + " with matchingContext should match",
+ matcher.match(matchingContext));
+ assertFalse(matcherName + " with mismatchingContext shouldn't match",
+ matcher.match(mismatchingContext));
+ assertFalse(matcherName + " with emptyContext shouldn't match",
+ matcher.match(emptyContext));
+ });
+ }
+
+ @Test
+ public void testVariableMatcherSubstitutions() {
+ VariableContext matchingContext = new VariableContext();
+ matchingContext.put("%user", "bob");
+ matchingContext.put("%primary_group", "developers");
+ matchingContext.put("%application", "TurboMR");
+ matchingContext.put("%custom", "bob");
+ matchingContext.put("%cus", "b");
+ matchingContext.put("%tom", "ob");
+
+ VariableContext mismatchingContext = new VariableContext();
+ mismatchingContext.put("%user", "dave");
+ mismatchingContext.put("%primary_group", "testers");
+ mismatchingContext.put("%application", "Tester APP");
+ mismatchingContext.put("%custom", "bob");
+ mismatchingContext.put("%cus", "b");
+ mismatchingContext.put("%tom", "ob");
+
+ MappingRuleMatcher customUser =
+ new MappingRuleMatchers.VariableMatcher("%custom", "%user");
+
+ MappingRuleMatcher userCusTom =
+ new MappingRuleMatchers.VariableMatcher("%user", "%cus%tom");
+
+ MappingRuleMatcher userCustom =
+ new MappingRuleMatchers.VariableMatcher("%user", "%custom");
+
+ MappingRuleMatcher userUser =
+ new MappingRuleMatchers.VariableMatcher("%user", "%user");
+
+ MappingRuleMatcher userStatic =
+ new MappingRuleMatchers.VariableMatcher("%user", "bob");
+
+ assertTrue("%custom should match %user in matching context",
+ customUser.match(matchingContext));
+ assertTrue("%user should match %custom in matching context",
+ userCustom.match(matchingContext));
+ assertTrue("%user (bob) should match %cus%tom (b + ob) in matching
context",
+ userCusTom.match(matchingContext));
+ assertTrue("%user should match %user in any context",
+ userUser.match(matchingContext));
+ assertTrue("%user (bob) should match bob in in matching context",
+ userStatic.match(matchingContext));
+
+ assertFalse(
+ "%custom (bob) should NOT match %user (dave) in mismatching context",
+ customUser.match(mismatchingContext));
+ assertFalse(
+ "%user (dave) should NOT match %custom (bob) in mismatching context",
+ userCustom.match(mismatchingContext));
+ assertFalse(
+ "%user (dave) should NOT match %cus%tom (b+ob) in mismatching context",
+ userCusTom.match(mismatchingContext));
+ assertTrue("%user should match %user in any context",
+ userUser.match(mismatchingContext));
+ assertFalse(
+ "%user (dave) should NOT match match bob in in matching context",
+ userStatic.match(mismatchingContext));
+ }
+
+ @Test
+ public void testVariableMatcherWithEmpties() {
+ VariableContext emptyContext = new VariableContext();
+ VariableContext allNull = new VariableContext();
+ allNull.put("%null", null);
+ allNull.put("%empty", null);
+
+ VariableContext allEmpty = new VariableContext();
+ allEmpty.put("%null", "");
+ allEmpty.put("%empty", "");
+
+ VariableContext allMakesSense = new VariableContext();
+ allMakesSense.put("%null", null);
+ allMakesSense.put("%empty", "");
+
+ VariableContext allFull = new VariableContext();
+ allFull.put("%null", "full");
+ allFull.put("%empty", "full");
+
+ MappingRuleMatcher nullMatcher =
+ new MappingRuleMatchers.VariableMatcher("%null", null);
+
+ MappingRuleMatcher emptyMatcher =
+ new MappingRuleMatchers.VariableMatcher("%empty", "");
+
+ //nulls should be handled as empty strings, so all should match
+ assertTrue(nullMatcher.match(emptyContext));
+ assertTrue(emptyMatcher.match(emptyContext));
+ assertTrue(nullMatcher.match(allEmpty));
+ assertTrue(emptyMatcher.match(allEmpty));
+ assertTrue(nullMatcher.match(allNull));
+ assertTrue(emptyMatcher.match(allNull));
+ assertTrue(nullMatcher.match(allMakesSense));
+ assertTrue(emptyMatcher.match(allMakesSense));
+
+ //neither null nor "" should match an actual string
+ assertFalse(nullMatcher.match(allFull));
+ assertFalse(emptyMatcher.match(allFull));
+
+ MappingRuleMatcher nullVarMatcher =
+ new MappingRuleMatchers.VariableMatcher(null, "");
+
+ //null variable should never match
+ assertFalse(nullVarMatcher.match(emptyContext));
+ assertFalse(nullVarMatcher.match(allNull));
+ assertFalse(nullVarMatcher.match(allEmpty));
+ assertFalse(nullVarMatcher.match(allMakesSense));
+ assertFalse(nullVarMatcher.match(allFull));
+ }
+
+ @Test
+ public void testBoolOperatorMatchers() {
+ VariableContext developerBob = new VariableContext();
+ developerBob.put("%user", "bob");
+ developerBob.put("%primary_group", "developers");
+
+
+ VariableContext testerBob = new VariableContext();
+ testerBob.put("%user", "bob");
+ testerBob.put("%primary_group", "testers");
+
+ VariableContext testerDave = new VariableContext();
+ testerDave.put("%user", "dave");
+ testerDave.put("%primary_group", "testers");
+
+
+ VariableContext accountantDave = new VariableContext();
+ accountantDave.put("%user", "dave");
+ accountantDave.put("%primary_group", "accountants");
+
+ MappingRuleMatcher userBob =
+ new MappingRuleMatchers.VariableMatcher("%user", "bob");
+
+ MappingRuleMatcher groupDevelopers =
+ new MappingRuleMatchers.VariableMatcher(
+ "%primary_group", "developers");
+
+ MappingRuleMatcher groupAccountants =
+ new MappingRuleMatchers.VariableMatcher(
+ "%primary_group", "accountants");
+
+ MappingRuleMatcher developerBobMatcher = new
MappingRuleMatchers.AndMatcher(
+ userBob, groupDevelopers);
+
+ MappingRuleMatcher testerDaveMatcher =
+ MappingRuleMatchers.createUserGroupMatcher("dave", "testers");
+
+ MappingRuleMatcher accountantOrBobMatcher =
+ new MappingRuleMatchers.OrMatcher(groupAccountants, userBob);
+
+ assertTrue(developerBobMatcher.match(developerBob));
+ assertFalse(developerBobMatcher.match(testerBob));
+ assertFalse(developerBobMatcher.match(testerDave));
+ assertFalse(developerBobMatcher.match(accountantDave));
+
+ assertFalse(testerDaveMatcher.match(developerBob));
+ assertFalse(testerDaveMatcher.match(testerBob));
+ assertTrue(testerDaveMatcher.match(testerDave));
+ assertFalse(testerDaveMatcher.match(accountantDave));
+
+ assertTrue(accountantOrBobMatcher.match(developerBob));
+ assertTrue(accountantOrBobMatcher.match(testerBob));
+ assertFalse(accountantOrBobMatcher.match(testerDave));
+ assertTrue(accountantOrBobMatcher.match(accountantDave));
+ }
+
+ @Test
+ public void testToStrings() {
+ MappingRuleMatcher var = new MappingRuleMatchers.VariableMatcher("%a",
"b");
+ MappingRuleMatcher all = MappingRuleMatchers.createAllMatcher();
+ MappingRuleMatcher and = new MappingRuleMatchers.AndMatcher(var, all, var);
+ MappingRuleMatcher or = new MappingRuleMatchers.OrMatcher(var, all, var);
+
+ assertEquals("VariableMatcher{variable='%a', value='b'}", var.toString());
+ assertEquals("MatchAllMatcher", all.toString());
+ assertEquals("AndMatcher{matchers=[" + var.toString() +
+ ", " + all.toString() +
+ ", " + var.toString() + "]}", and.toString());
+ assertEquals("OrMatcher{matchers=[" + var.toString() +
+ ", " + all.toString() +
+ ", " + var.toString() + "]}", or.toString());
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
