Repository: tez
Updated Branches:
refs/heads/branch-0.7 067783297 -> 1e09d072c
TEZ-2949. Allow duplicate dag names within session for Tez. (hitesh)
(cherry picked from commit 9024cade51d4ad4ea8e2b54e27e8fac6664009a8)
Conflicts:
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1e09d072
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1e09d072
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1e09d072
Branch: refs/heads/branch-0.7
Commit: 1e09d072c22ba2ab4f2b4882eb8a89c5a0013774
Parents: 0677832
Author: Hitesh Shah <[email protected]>
Authored: Fri Nov 20 13:39:05 2015 -0800
Committer: Hitesh Shah <[email protected]>
Committed: Fri Nov 20 13:41:01 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 4 --
.../org/apache/tez/dag/app/RecoveryParser.java | 3 --
.../apache/tez/dag/app/TestRecoveryParser.java | 1 -
.../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 52 --------------------
.../org/apache/tez/test/TestDAGRecovery.java | 8 ---
6 files changed, 1 insertion(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/1e09d072/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 26e4913..df05770 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
TEZ-2679. Admin forms of launch env settings
ALL CHANGES
+ TEZ-2949. Allow duplicate dag names within session for Tez.
TEZ-2933. Tez UI: Load application details from RM when available
TEZ-2908. Tez UI: Errors are logged, but not displayed in the UI when AM
fetch fails
TEZ-2923. Tez Live UI counters view empty for vertices, tasks, attempts
http://git-wip-us.apache.org/repos/asf/tez/blob/1e09d072/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 2ea086d..f268b5e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -2204,10 +2204,6 @@ public class DAGAppMaster extends AbstractService {
throws TezException {
long submitTime = this.clock.getTime();
this.appName = dagPlan.getName();
- if (dagNames.contains(dagPlan.getName())) {
- throw new TezException("Duplicate dag name '" + dagPlan.getName() + "'");
- }
- dagNames.add(dagPlan.getName());
// /////////////////// Create the job itself.
DAG newDAG = createDAG(dagPlan);
http://git-wip-us.apache.org/repos/asf/tez/blob/1e09d072/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
index 11d126a..3641dcc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
@@ -323,7 +323,6 @@ public class RecoveryParser {
static class DAGSummaryData {
final TezDAGID dagId;
- String dagName;
boolean completed = false;
boolean dagCommitCompleted = true;
DAGState dagState;
@@ -346,7 +345,6 @@ public class RecoveryParser {
completed = false;
DAGSubmittedEvent dagSubmittedEvent = new DAGSubmittedEvent();
dagSubmittedEvent.fromSummaryProtoStream(proto);
- dagName = dagSubmittedEvent.getDAGName();
break;
case DAG_FINISHED:
completed = true;
@@ -541,7 +539,6 @@ public class RecoveryParser {
// Set counter for next set of DAGs & update dagNames Set in DAGAppMaster
dagAppMaster.setDAGCounter(dagCounter);
for (DAGSummaryData dagSummaryData: dagSummaryDataMap.values()){
- dagAppMaster.dagNames.add(dagSummaryData.dagName);
dagAppMaster.dagIDs.add(dagSummaryData.dagId.toString());
}
http://git-wip-us.apache.org/repos/asf/tez/blob/1e09d072/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
----------------------------------------------------------------------
diff --git
a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
index e261df6..be67cb2 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
@@ -70,7 +70,6 @@ public class TestRecoveryParser {
this.recoveryPath = new Path(TEST_ROOT_DIR + "/" + appId + "/recovery");
this.localFS.delete(new Path(TEST_ROOT_DIR), true);
mockAppMaster = mock(DAGAppMaster.class);
- mockAppMaster.dagNames = new HashSet<String>();
mockAppMaster.dagIDs = new HashSet<String>();
when(mockAppMaster.getConfig()).thenReturn(new Configuration());
mockDAGImpl = mock(DAGImpl.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/1e09d072/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git
a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index db1b1e1..92f70c5 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -20,8 +20,6 @@ package org.apache.tez.mapreduce;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
@@ -222,56 +220,6 @@ public class TestMRRJobsDAGApi {
tezSession.stop();
}
- @Test(timeout = 100000)
- public void testMultipleDAGsWithDuplicateName() throws TezException,
IOException,
- InterruptedException {
- TezClient tezSession = null;
- try {
- TezConfiguration tezConf = new
TezConfiguration(mrrTezCluster.getConfig());
- Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp",
String.valueOf(random
- .nextInt(100000))));
- remoteFs.mkdirs(remoteStagingDir);
- tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
remoteStagingDir.toString());
- tezSession = TezClient.create("OrderedWordCountSession", tezConf, true);
- tezSession.start();
-
- SleepProcessorConfig spConf = new SleepProcessorConfig(1);
- for (int dagIndex = 1; dagIndex <= 2; dagIndex++) {
- DAG dag = DAG.create("TezSleepProcessor");
- Vertex vertex = Vertex.create("SleepVertex",
ProcessorDescriptor.create(
-
SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
- Resource.newInstance(1024, 1));
- dag.addVertex(vertex);
-
- DAGClient dagClient = null;
- try {
- dagClient = tezSession.submitDAG(dag);
- if (dagIndex > 1) {
- fail("Should fail due to duplicate dag name for dagIndex: " +
dagIndex);
- }
- } catch (TezException tex) {
- if (dagIndex > 1) {
- assertTrue(tex.getMessage().contains("Duplicate dag name "));
- continue;
- }
- fail("DuplicateDAGName exception thrown for 1st DAG submission");
- }
- DAGStatus dagStatus = dagClient.getDAGStatus(null);
- while (!dagStatus.isCompleted()) {
- LOG.debug("Waiting for job to complete. Sleeping for 500ms." + "
Current state: "
- + dagStatus.getState());
- Thread.sleep(500l);
- dagStatus = dagClient.getDAGStatus(null);
- }
- }
- } finally {
- if (tezSession != null) {
- tezSession.stop();
- }
- }
- }
-
-
@Test(timeout = 60000)
public void testNonDefaultFSStagingDir() throws Exception {
SleepProcessorConfig spConf = new SleepProcessorConfig(1);
http://git-wip-us.apache.org/repos/asf/tez/blob/1e09d072/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
index 74c2727..778825b 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
@@ -238,14 +238,6 @@ public class TestDAGRecovery {
runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
verifyRecoveryLog();
-
- // it should fail if submitting same dags in recovery mode (TEZ-1064)
- try {
- DAGClient dagClient = tezSession.submitDAG(dag);
- Assert.fail("Expected DAG submit to fail on duplicate dag name");
- } catch (TezException e) {
- Assert.assertTrue(e.getMessage().contains("Duplicate dag name"));
- }
}
@Test(timeout=120000)
