Repository: tez Updated Branches: refs/heads/master 7ee716517 -> 54a429e0b
TEZ-1325. RecoveryParser can find incorrect last DAG ID. (Jeff Zhang via hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/54a429e0 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/54a429e0 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/54a429e0 Branch: refs/heads/master Commit: 54a429e0bbe64861fdc569fe9c9a39e74b187c80 Parents: 7ee7165 Author: Hitesh Shah <[email protected]> Authored: Tue Aug 19 17:29:36 2014 -0700 Committer: Hitesh Shah <[email protected]> Committed: Tue Aug 19 17:29:57 2014 -0700 ---------------------------------------------------------------------- .../org/apache/tez/dag/app/RecoveryParser.java | 14 ++- .../apache/tez/dag/app/TestRecoveryParser.java | 95 ++++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/54a429e0/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java index de9979d..715d4e0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java @@ -71,6 +71,8 @@ import org.apache.tez.dag.records.TezVertexID; import org.apache.tez.dag.recovery.records.RecoveryProtos; import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; +import com.google.common.annotations.VisibleForTesting; + public class RecoveryParser { private static final Log LOG = LogFactory.getLog(RecoveryParser.class); @@ -307,7 +309,8 @@ public class RecoveryParser { return recoveryFS.create(dagRecoveryPath, true, recoveryBufferSize); } - private DAGSummaryData getLastCompletedOrInProgressDAG( + @VisibleForTesting + DAGSummaryData getLastCompletedOrInProgressDAG( Map<TezDAGID, DAGSummaryData> dagSummaryDataMap) { DAGSummaryData inProgressDAG = null; DAGSummaryData lastCompletedDAG = null; @@ -320,7 +323,10 @@ public class RecoveryParser { } inProgressDAG = entry.getValue(); } else { - lastCompletedDAG = entry.getValue(); + if (lastCompletedDAG == null || + lastCompletedDAG.dagId.getId() < entry.getValue().dagId.getId()) { + lastCompletedDAG = entry.getValue(); + } } } if (inProgressDAG == null) { @@ -388,7 +394,8 @@ public class RecoveryParser { return TezCommonUtils.getAttemptRecoveryPath(recoveryDataDir, foundPreviousAttempt); } - private static class DAGSummaryData { + @VisibleForTesting + static class DAGSummaryData { final TezDAGID dagId; String dagName; @@ -546,6 +553,7 @@ public class RecoveryParser { + ", lastModTime=" + summaryFileStatus.getModificationTime()); int dagCounter = 0; + Map<TezDAGID, DAGSummaryData> dagSummaryDataMap = new HashMap<TezDAGID, DAGSummaryData>(); while (true) { http://git-wip-us.apache.org/repos/asf/tez/blob/54a429e0/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java new file mode 100644 index 0000000..77c76c5 --- /dev/null +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.dag.app; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.tez.dag.app.RecoveryParser.DAGSummaryData; +import org.apache.tez.dag.records.TezDAGID; +import org.junit.*; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +public class TestRecoveryParser { + + private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + + TestRecoveryParser.class.getName() + "-tmpDir"; + + private RecoveryParser parser; + + @Before + public void setUp() throws IllegalArgumentException, IOException { + DAGAppMaster mockAppMaster = mock(DAGAppMaster.class); + when(mockAppMaster.getConfig()).thenReturn(new Configuration()); + parser = + new RecoveryParser(mockAppMaster, mock(FileSystem.class), new Path( + TEST_ROOT_DIR), 3); + } + + private DAGSummaryData createDAGSummaryData(TezDAGID dagId, boolean completed) { + DAGSummaryData data = new DAGSummaryData(dagId); + data.completed = completed; + return data; + } + + @Test + public void testGetLastCompletedDAG() { + Map<TezDAGID, DAGSummaryData> summaryDataMap = + new HashMap<TezDAGID, DAGSummaryData>(); + int lastCompletedDAGId = new Random().nextInt(100); + for (int i = 1; i <= lastCompletedDAGId; ++i) { + ApplicationId appId = ApplicationId.newInstance(1, 1); + TezDAGID dagId = TezDAGID.getInstance(appId, i); + summaryDataMap.put(dagId, createDAGSummaryData(dagId, true)); + } + + DAGSummaryData lastCompletedDAG = + parser.getLastCompletedOrInProgressDAG(summaryDataMap); + assertEquals(lastCompletedDAGId, lastCompletedDAG.dagId.getId()); + } + + @Test + public void testGetLastInProgressDAG() { + Map<TezDAGID, DAGSummaryData> summaryDataMap = + new HashMap<TezDAGID, DAGSummaryData>(); + int dagNum = 100; + int lastInProgressDAGId = new Random().nextInt(dagNum); + for (int i = 1; i <= dagNum; ++i) { + ApplicationId appId = ApplicationId.newInstance(1, 1); + TezDAGID dagId = TezDAGID.getInstance(appId, i); + if (i == lastInProgressDAGId) { + summaryDataMap.put(dagId, createDAGSummaryData(dagId, false)); + } else { + summaryDataMap.put(dagId, createDAGSummaryData(dagId, true)); + } + } + + DAGSummaryData lastInProgressDAG = + parser.getLastCompletedOrInProgressDAG(summaryDataMap); + assertEquals(lastInProgressDAGId, lastInProgressDAG.dagId.getId()); + } +}
