Repository: tez Updated Branches: refs/heads/master 4561b8252 -> 9024cade5
TEZ-2949. Allow duplicate dag names within session for Tez. (hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9024cade Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9024cade Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9024cade Branch: refs/heads/master Commit: 9024cade51d4ad4ea8e2b54e27e8fac6664009a8 Parents: 4561b82 Author: Hitesh Shah <[email protected]> Authored: Fri Nov 20 13:39:05 2015 -0800 Committer: Hitesh Shah <[email protected]> Committed: Fri Nov 20 13:39:05 2015 -0800 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../org/apache/tez/dag/app/DAGAppMaster.java | 4 -- .../org/apache/tez/dag/app/RecoveryParser.java | 3 -- .../apache/tez/dag/app/TestRecoveryParser.java | 1 - .../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 52 -------------------- .../org/apache/tez/test/TestDAGRecovery.java | 8 --- 6 files changed, 2 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 9644a1e..223dcc1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,6 +8,7 @@ INCOMPATIBLE CHANGES TEZ-2948. Stop using dagName in the dagComplete notification to TaskCommunicators. ALL CHANGES: + TEZ-2949. Allow duplicate dag names within session for Tez. TEZ-2952. NPE in TestOnFileUnorderedKVOutput TEZ-2480. Exception when closing output is ignored. TEZ-2944. NPE in TestProcessorContext. @@ -259,6 +260,7 @@ INCOMPATIBLE CHANGES TEZ-2679. Admin forms of launch env settings ALL CHANGES + TEZ-2949. Allow duplicate dag names within session for Tez. TEZ-2923. Tez Live UI counters view empty for vertices, tasks, attempts TEZ-2935. Add MR slow start translation for ShuffleVertexManager TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 9c09f95..2c50264 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2298,10 +2298,6 @@ public class DAGAppMaster extends AbstractService { throws TezException { long submitTime = this.clock.getTime(); this.appName = dagPlan.getName(); - if (dagNames.contains(dagPlan.getName())) { - throw new TezException("Duplicate dag name '" + dagPlan.getName() + "'"); - } - dagNames.add(dagPlan.getName()); // /////////////////// Create the job itself. DAG newDAG = createDAG(dagPlan); http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java index 8ce835b..046dbd9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java @@ -323,7 +323,6 @@ public class RecoveryParser { static class DAGSummaryData { final TezDAGID dagId; - String dagName; boolean completed = false; boolean dagCommitCompleted = true; DAGState dagState; @@ -346,7 +345,6 @@ public class RecoveryParser { completed = false; DAGSubmittedEvent dagSubmittedEvent = new DAGSubmittedEvent(); dagSubmittedEvent.fromSummaryProtoStream(proto); - dagName = dagSubmittedEvent.getDAGName(); break; case DAG_FINISHED: completed = true; @@ -543,7 +541,6 @@ public class RecoveryParser { // Set counter for next set of DAGs & update dagNames Set in DAGAppMaster dagAppMaster.setDAGCounter(dagCounter); for (DAGSummaryData dagSummaryData: dagSummaryDataMap.values()){ - dagAppMaster.dagNames.add(dagSummaryData.dagName); dagAppMaster.dagIDs.add(dagSummaryData.dagId.toString()); } http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java index e261df6..be67cb2 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java @@ -70,7 +70,6 @@ public class TestRecoveryParser { this.recoveryPath = new Path(TEST_ROOT_DIR + "/" + appId + "/recovery"); this.localFS.delete(new Path(TEST_ROOT_DIR), true); mockAppMaster = mock(DAGAppMaster.class); - mockAppMaster.dagNames = new HashSet<String>(); mockAppMaster.dagIDs = new HashSet<String>(); when(mockAppMaster.getConfig()).thenReturn(new Configuration()); mockDAGImpl = mock(DAGImpl.class); http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java ---------------------------------------------------------------------- diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java index db1b1e1..92f70c5 100644 --- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java +++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java @@ -20,8 +20,6 @@ package org.apache.tez.mapreduce; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; @@ -222,56 +220,6 @@ public class TestMRRJobsDAGApi { tezSession.stop(); } - @Test(timeout = 100000) - public void testMultipleDAGsWithDuplicateName() throws TezException, IOException, - InterruptedException { - TezClient tezSession = null; - try { - TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); - Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random - .nextInt(100000)))); - remoteFs.mkdirs(remoteStagingDir); - tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); - tezSession = TezClient.create("OrderedWordCountSession", tezConf, true); - tezSession.start(); - - SleepProcessorConfig spConf = new SleepProcessorConfig(1); - for (int dagIndex = 1; dagIndex <= 2; dagIndex++) { - DAG dag = DAG.create("TezSleepProcessor"); - Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create( - SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, - Resource.newInstance(1024, 1)); - dag.addVertex(vertex); - - DAGClient dagClient = null; - try { - dagClient = tezSession.submitDAG(dag); - if (dagIndex > 1) { - fail("Should fail due to duplicate dag name for dagIndex: " + dagIndex); - } - } catch (TezException tex) { - if (dagIndex > 1) { - assertTrue(tex.getMessage().contains("Duplicate dag name ")); - continue; - } - fail("DuplicateDAGName exception thrown for 1st DAG submission"); - } - DAGStatus dagStatus = dagClient.getDAGStatus(null); - while (!dagStatus.isCompleted()) { - LOG.debug("Waiting for job to complete. Sleeping for 500ms." + " Current state: " - + dagStatus.getState()); - Thread.sleep(500l); - dagStatus = dagClient.getDAGStatus(null); - } - } - } finally { - if (tezSession != null) { - tezSession.stop(); - } - } - } - - @Test(timeout = 60000) public void testNonDefaultFSStagingDir() throws Exception { SleepProcessorConfig spConf = new SleepProcessorConfig(1); http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java ---------------------------------------------------------------------- diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java index 74c2727..778825b 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java @@ -238,14 +238,6 @@ public class TestDAGRecovery { runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); verifyRecoveryLog(); - - // it should fail if submitting same dags in recovery mode (TEZ-1064) - try { - DAGClient dagClient = tezSession.submitDAG(dag); - Assert.fail("Expected DAG submit to fail on duplicate dag name"); - } catch (TezException e) { - Assert.assertTrue(e.getMessage().contains("Duplicate dag name")); - } } @Test(timeout=120000)
