Repository: tez Updated Branches: refs/heads/master 3894c5ec6 -> 2382f09ff
TEZ-2221. VertexGroup name should be unqiue (zjffdu) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2382f09f Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2382f09f Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2382f09f Branch: refs/heads/master Commit: 2382f09ffaabe1be6ede8203025027198c3e6d2b Parents: 3894c5e Author: Jeff Zhang <[email protected]> Authored: Thu Apr 30 12:53:30 2015 +0800 Committer: Jeff Zhang <[email protected]> Committed: Thu Apr 30 12:53:30 2015 +0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../main/java/org/apache/tez/dag/api/DAG.java | 22 +++++++++++++++++++- .../org/apache/tez/dag/api/VertexGroup.java | 10 ++------- .../java/org/apache/tez/dag/api/TestDAG.java | 16 ++++++++++++++ 4 files changed, 40 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/2382f09f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index cfdc679..4b031ca 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -325,6 +325,7 @@ TEZ-UI CHANGES (TEZ-8): Release 0.5.4: Unreleased ALL CHANGES: + TEZ-2221. VertexGroup name should be unqiue TEZ-1521. VertexDataMovementEventsGeneratedEvent may be logged twice in recovery log TEZ-2348. EOF exception during UnorderedKVReader.next(). TEZ-1560. Invalid state machine handling for V_SOURCE_VERTEX_RECOVERED in recovery. http://git-wip-us.apache.org/repos/asf/tez/blob/2382f09f/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java index 79e8759..92f865d 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java @@ -60,7 +60,9 @@ import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType; import org.apache.tez.dag.api.records.DAGProtos.VertexPlan; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -83,6 +85,9 @@ public class DAG { final Collection<URI> urisForCredentials = new HashSet<URI>(); Credentials credentials = new Credentials(); Set<VertexGroup> vertexGroups = Sets.newHashSet(); + // to verify the vertex Group memberSet should be unique + private Set<Set<String>> vertexGroupMemberSets = Sets.newHashSet(); + Set<GroupInputEdge> groupInputEdges = Sets.newHashSet(); private DAGAccessControls dagAccessControls; @@ -176,11 +181,26 @@ public class DAG { * @return {@link DAG} */ public synchronized VertexGroup createVertexGroup(String name, Vertex... members) { + // vertex group member set should be unique + Collection<String> memberNames = + Collections2.transform(Lists.newArrayList(members), new Function<Vertex, String>() { + @Override + public String apply(Vertex v) { + return v.getName(); + } + }); + if (!vertexGroupMemberSets.add(Sets.newHashSet(memberNames))){ + throw new IllegalStateException( + "VertexGroup " + memberNames + " already defined as another group!"); + } + + // vertex group name should be unique. VertexGroup uv = new VertexGroup(name, members); - if (!vertexGroups.add(uv)){ + if (!vertexGroups.add(uv)) { throw new IllegalStateException( "VertexGroup " + name + " already defined!"); } + return uv; } http://git-wip-us.apache.org/repos/asf/tez/blob/2382f09f/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java index 84a822f..8041b34 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java @@ -108,7 +108,7 @@ public class VertexGroup { @Override public String toString() { - return "[ VertexGroup: " + groupInfo.getGroupName() + "]"; + return "[ VertexGroup: " + groupInfo.getGroupName() + "]" + ", members=" + groupInfo.members; } GroupInfo getGroupInfo() { @@ -129,8 +129,6 @@ public class VertexGroup { int result = 1; result = prime * result + ((groupInfo.groupName == null) ? 0 : groupInfo.groupName.hashCode()); - result = prime * result - + ((groupInfo.members == null) ? 0 : groupInfo.members.hashCode()); return result; } @@ -148,11 +146,7 @@ public class VertexGroup { return false; } else if (!groupInfo.groupName.equals(other.groupInfo.groupName)) return false; - if (groupInfo.members == null) { - if (other.groupInfo.members != null) - return false; - } else if (!groupInfo.members.equals(other.groupInfo.members)) - return false; + return true; } } http://git-wip-us.apache.org/repos/asf/tez/blob/2382f09f/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java index 0590907..245d9c0 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java @@ -82,6 +82,8 @@ public class TestDAG { dummyTaskCount, dummyTaskResource); Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); + Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Processor"), + dummyTaskCount, dummyTaskResource); DAG dag = DAG.create("testDAG"); dag.createVertexGroup("group_1", v1,v2); @@ -90,8 +92,22 @@ public class TestDAG { Assert.fail("should fail it due to duplicated VertexGroups"); } catch (Exception e) { e.printStackTrace(); + Assert.assertEquals("VertexGroup [v1, v2] already defined as another group!", e.getMessage()); + } + try { + dag.createVertexGroup("group_1", v2, v3); + Assert.fail("should fail it due to duplicated VertexGroups"); + } catch (Exception e) { + e.printStackTrace(); Assert.assertEquals("VertexGroup group_1 already defined!", e.getMessage()); } + try { + dag.createVertexGroup("group_2", v1, v2); + Assert.fail("should fail it due to duplicated VertexGroups"); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals("VertexGroup [v1, v2] already defined as another group!", e.getMessage()); + } } @Test(timeout = 5000)
