Repository: hadoop Updated Branches: refs/heads/branch-3.1 33f82323b -> eb8765bbe refs/heads/trunk 451265a83 -> e85188101
YARN-7921. Transform a PlacementConstraint to a string expression. Contributed by Weiwei Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8518810 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8518810 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8518810 Branch: refs/heads/trunk Commit: e85188101c6c74b348a2fb6aa0f4e85c81b4a28c Parents: 451265a Author: Konstantinos Karanasos <kkarana...@apache.org> Authored: Mon Feb 26 12:15:16 2018 -0800 Committer: Konstantinos Karanasos <kkarana...@apache.org> Committed: Mon Feb 26 12:15:16 2018 -0800 ---------------------------------------------------------------------- .../yarn/api/resource/PlacementConstraint.java | 141 ++++++++++++++++++- .../resource/TestPlacementConstraintParser.java | 102 ++++++++++++-- .../TestPlacementConstraintTransformations.java | 7 + 3 files changed, 235 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8518810/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java index c054cbc..9bb17f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -23,6 +23,8 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.Iterator; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -45,6 +47,11 @@ public class PlacementConstraint { this.constraintExpr = constraintExpr; } + @Override + public String toString() { + return this.constraintExpr.toString(); + } + /** * Get the constraint expression of the placement constraint. * @@ -226,6 +233,42 @@ public class PlacementConstraint { } @Override + public String toString() { + int max = getMaxCardinality(); + int min = getMinCardinality(); + List<String> targetExprList = getTargetExpressions().stream() + .map(TargetExpression::toString).collect(Collectors.toList()); + List<String> targetConstraints = new ArrayList<>(); + for (String targetExpr : targetExprList) { + if (min == 0 && max == 0) { + // anti-affinity + targetConstraints.add(new StringBuilder() + .append("notin").append(",") + .append(getScope()).append(",") + .append(targetExpr) + .toString()); + } else if (min == 1 && max == Integer.MAX_VALUE) { + // affinity + targetConstraints.add(new StringBuilder() + .append("in").append(",") + .append(getScope()).append(",") + .append(targetExpr) + .toString()); + } else { + // cardinality + targetConstraints.add(new StringBuilder() + .append("cardinality").append(",") + .append(getScope()).append(",") + .append(targetExpr).append(",") + .append(min).append(",") + .append(max) + .toString()); + } + } + return String.join(":", targetConstraints); + } + + @Override public <T> T accept(Visitor<T> visitor) { return visitor.visit(this); } @@ -326,6 +369,23 @@ public class PlacementConstraint { } @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + if (TargetType.ALLOCATION_TAG == this.targetType) { + // following by a comma separated tags + sb.append(String.join(",", getTargetValues())); + } else if (TargetType.NODE_ATTRIBUTE == this.targetType) { + // following by a comma separated key value pairs + if (this.getTargetValues() != null) { + String attributeName = this.getTargetKey(); + String attributeValues = String.join(":", this.getTargetValues()); + sb.append(attributeName + "=[" + attributeValues + "]"); + } + } + return sb.toString(); + } + + @Override public <T> T accept(Visitor<T> visitor) { return visitor.visit(this); } @@ -345,7 +405,16 @@ public class PlacementConstraint { * TargetOperator enum helps to specify type. */ enum TargetOperator { - IN, NOT_IN + IN("in"), NOT_IN("notin"); + + private String operator; + TargetOperator(String op) { + this.operator = op; + } + + String getOperator() { + return this.operator; + } } private TargetOperator op; @@ -415,6 +484,17 @@ public class PlacementConstraint { } @Override + public String toString() { + List<String> targetExprs = getTargetExpressions().stream().map( + targetExpression -> new StringBuilder() + .append(op.getOperator()).append(",") + .append(scope).append(",") + .append(targetExpression.toString()) + .toString()).collect(Collectors.toList()); + return String.join(":", targetExprs); + } + + @Override public <T> T accept(Visitor<T> visitor) { return visitor.visit(this); } @@ -517,6 +597,17 @@ public class PlacementConstraint { + (allocationTags != null ? allocationTags.hashCode() : 0); return result; } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("cardinality").append(",").append(getScope()).append(","); + for (String tag : getAllocationTags()) { + sb.append(tag).append(","); + } + sb.append(minCardinality).append(",").append(maxCardinality); + return sb.toString(); + } } /** @@ -580,6 +671,22 @@ public class PlacementConstraint { public <T> T accept(Visitor<T> visitor) { return visitor.visit(this); } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("and("); + Iterator<AbstractConstraint> it = getChildren().iterator(); + while (it.hasNext()) { + AbstractConstraint child = it.next(); + sb.append(child.toString()); + if (it.hasNext()) { + sb.append(":"); + } + } + sb.append(")"); + return sb.toString(); + } } /** @@ -606,6 +713,22 @@ public class PlacementConstraint { public <T> T accept(Visitor<T> visitor) { return visitor.visit(this); } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("or("); + Iterator<AbstractConstraint> it = getChildren().iterator(); + while (it.hasNext()) { + AbstractConstraint child = it.next(); + sb.append(child.toString()); + if (it.hasNext()) { + sb.append(":"); + } + } + sb.append(")"); + return sb.toString(); + } } /** @@ -636,6 +759,22 @@ public class PlacementConstraint { public <T> T accept(Visitor<T> visitor) { return visitor.visit(this); } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("DelayedOr("); + Iterator<TimedPlacementConstraint> it = getChildren().iterator(); + while (it.hasNext()) { + TimedPlacementConstraint child = it.next(); + sb.append(child.toString()); + if (it.hasNext()) { + sb.append(","); + } + } + sb.append(")"); + return sb.toString(); + } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8518810/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java index 941f971..a69571c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Set; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; import org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException; @@ -51,42 +52,50 @@ public class TestPlacementConstraintParser { @Test public void testTargetExpressionParser() throws PlacementConstraintParseException { + String expressionStr; ConstraintParser parser; AbstractConstraint constraint; SingleConstraint single; // Anti-affinity with single target tag - // NOTIN,NDOE,foo - parser = new TargetConstraintParser("NOTIN, NODE, foo"); + // NOTIN,NODE,foo + expressionStr = "NOTIN, NODE, foo"; + parser = new TargetConstraintParser(expressionStr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(0, single.getMaxCardinality()); + verifyConstraintToString(expressionStr, constraint); // lower cases is also valid - parser = new TargetConstraintParser("notin, node, foo"); + expressionStr = "notin, node, foo"; + parser = new TargetConstraintParser(expressionStr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(0, single.getMaxCardinality()); + verifyConstraintToString(expressionStr, constraint); // Affinity with single target tag // IN,NODE,foo - parser = new TargetConstraintParser("IN, NODE, foo"); + expressionStr = "IN, NODE, foo"; + parser = new TargetConstraintParser(expressionStr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(1, single.getMinCardinality()); Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality()); + verifyConstraintToString(expressionStr, constraint); // Anti-affinity with multiple target tags // NOTIN,NDOE,foo,bar,exp - parser = new TargetConstraintParser("NOTIN, NODE, foo, bar, exp"); + expressionStr = "NOTIN, NODE, foo, bar, exp"; + parser = new TargetConstraintParser(expressionStr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; @@ -98,6 +107,7 @@ public class TestPlacementConstraintParser { single.getTargetExpressions().iterator().next(); Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString()); Assert.assertEquals(3, exp.getTargetValues().size()); + verifyConstraintToString(expressionStr, constraint); // Invalid OP parser = new TargetConstraintParser("XYZ, NODE, foo"); @@ -112,12 +122,14 @@ public class TestPlacementConstraintParser { @Test public void testCardinalityConstraintParser() throws PlacementConstraintParseException { + String expressionExpr; ConstraintParser parser; AbstractConstraint constraint; SingleConstraint single; // cardinality,NODE,foo,0,1 - parser = new CardinalityConstraintParser("cardinality, NODE, foo, 0, 1"); + expressionExpr = "cardinality, NODE, foo, 0, 1"; + parser = new CardinalityConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; @@ -130,10 +142,11 @@ public class TestPlacementConstraintParser { Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString()); Assert.assertEquals(1, exp.getTargetValues().size()); Assert.assertEquals("foo", exp.getTargetValues().iterator().next()); + verifyConstraintToString(expressionExpr, constraint); // cardinality,NODE,foo,bar,moo,0,1 - parser = new CardinalityConstraintParser( - "cardinality,RACK,foo,bar,moo,0,1"); + expressionExpr = "cardinality,RACK,foo,bar,moo,0,1"; + parser = new CardinalityConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; @@ -147,6 +160,7 @@ public class TestPlacementConstraintParser { Set<String> expectedTags = Sets.newHashSet("foo", "bar", "moo"); Assert.assertTrue(Sets.difference(expectedTags, exp.getTargetValues()) .isEmpty()); + verifyConstraintToString(expressionExpr, constraint); // Invalid scope string try { @@ -174,25 +188,29 @@ public class TestPlacementConstraintParser { @Test public void testAndConstraintParser() throws PlacementConstraintParseException { + String expressionExpr; ConstraintParser parser; AbstractConstraint constraint; And and; - parser = new ConjunctionConstraintParser( - "AND(NOTIN,NODE,foo:NOTIN,NODE,bar)"); + expressionExpr = "AND(NOTIN,NODE,foo:NOTIN,NODE,bar)"; + parser = new ConjunctionConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof And); and = (And) constraint; Assert.assertEquals(2, and.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); - parser = new ConjunctionConstraintParser( - "AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)"); + expressionExpr = "AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)"; + parser = new ConjunctionConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof And); Assert.assertEquals(2, and.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); - parser = new ConjunctionConstraintParser( - "AND(NOTIN,NODE,foo:AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))"); + expressionExpr = + "AND(NOTIN,NODE,foo:AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))"; + parser = new ConjunctionConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof And); and = (And) constraint; @@ -200,6 +218,43 @@ public class TestPlacementConstraintParser { Assert.assertTrue(and.getChildren().get(1) instanceof And); and = (And) and.getChildren().get(1); Assert.assertEquals(2, and.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); + } + + @Test + public void testOrConstraintParser() + throws PlacementConstraintParseException { + String expressionExpr; + ConstraintParser parser; + AbstractConstraint constraint; + Or or; + + expressionExpr = "OR(NOTIN,NODE,foo:NOTIN,NODE,bar)"; + parser = new ConjunctionConstraintParser(expressionExpr); + constraint = parser.parse(); + Assert.assertTrue(constraint instanceof Or); + or = (Or) constraint; + Assert.assertEquals(2, or.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); + + expressionExpr = "OR(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)"; + parser = new ConjunctionConstraintParser(expressionExpr); + constraint = parser.parse(); + Assert.assertTrue(constraint instanceof Or); + Assert.assertEquals(2, or.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); + + expressionExpr = + "OR(NOTIN,NODE,foo:OR(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))"; + parser = new ConjunctionConstraintParser(expressionExpr); + constraint = parser.parse(); + Assert.assertTrue(constraint instanceof Or); + or = (Or) constraint; + Assert.assertTrue(or.getChildren().get(0) instanceof SingleConstraint); + Assert.assertTrue(or.getChildren().get(1) instanceof Or); + or = (Or) or.getChildren().get(1); + Assert.assertEquals(2, or.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); } @Test @@ -369,4 +424,23 @@ public class TestPlacementConstraintParser { expectedPc2 = targetNotIn("node", allocationTag("foo")).build(); Assert.assertEquals(expectedPc2, actualPc2); } + + // We verify the toString result by parsing it again + // instead of raw string comparing. This is because internally + // we are not storing tags strictly to its original order, so + // the toString result might have different ordering with the + // input expression. + private void verifyConstraintToString(String inputExpr, + AbstractConstraint constraint) { + String constrainExpr = constraint.toString(); + System.out.println("Input: " + inputExpr + .toLowerCase().replaceAll(" ", "")); + System.out.println("ToString: " + constrainExpr); + try { + PlacementConstraintParser.parseExpression(constrainExpr); + } catch (PlacementConstraintParseException e) { + Assert.fail("The parser is unable to parse the expression: " + + constrainExpr + ", caused by: " + e.getMessage()); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8518810/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java index aa92d7a..557b82f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java @@ -65,6 +65,10 @@ public class TestPlacementConstraintTransformations { SingleConstraint single = (SingleConstraint) sConstraintExpr; TargetConstraint target = (TargetConstraint) tConstraintExpr; + + // Make sure the expression string is consistent + // before and after transforming + Assert.assertEquals(single.toString(), target.toString()); Assert.assertEquals(single.getScope(), target.getScope()); Assert.assertEquals(TargetOperator.IN, target.getOp()); Assert.assertEquals(single.getTargetExpressions(), @@ -101,6 +105,9 @@ public class TestPlacementConstraintTransformations { Assert.assertTrue(sConstraintExpr instanceof SingleConstraint); SingleConstraint single = (SingleConstraint) sConstraintExpr; + // Make sure the consistent expression string is consistent + // before and after transforming + Assert.assertEquals(single.toString(), cardinality.toString()); Assert.assertEquals(cardinality.getScope(), single.getScope()); Assert.assertEquals(cardinality.getMinCardinality(), single.getMinCardinality()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org