Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 ed512645b -> b8b67e4b8


MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko

(cherry picked from commit 08feac4c83939f60289f8540d72ffcabb7852e2a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8b67e4b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8b67e4b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8b67e4b

Branch: refs/heads/branch-2.9
Commit: b8b67e4b8da02dd99ab222c9548a5959acfd4df8
Parents: ed51264
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Feb 12 14:30:42 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Feb 12 14:33:32 2018 -0600

----------------------------------------------------------------------
 .../java/org/apache/hadoop/mapred/Task.java     | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 84 ++++++++++++++++++++
 2 files changed, 95 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8b67e4b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 169164f..a9ac184 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -195,6 +195,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey tokenSecret;
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
+  private boolean uberized = false;
 
   ////////////////////////////////////////////
   // Constructors
@@ -785,9 +786,6 @@ abstract public class Task implements Writable, 
Configurable {
       long taskProgressInterval = MRJobConfUtil.
           getTaskProgressReportInterval(conf);
 
-      boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-          false);
-
       while (!taskDone.get()) {
         synchronized (lock) {
           done = false;
@@ -1164,11 +1162,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
     int retries = MAX_RETRIES;
+
     while (true) {
       try {
         if (!umbilical.statusUpdate(getTaskID(), taskStatus)) {
-          LOG.warn("Parent died.  Exiting "+taskId);
-          System.exit(66);
+          if (uberized) {
+            LOG.warn("Task no longer available: " + taskId);
+            break;
+          } else {
+            LOG.warn("Parent died.  Exiting " + taskId);
+            ExitUtil.terminate(66);
+          }
         }
         taskStatus.clearStatus();
         return;
@@ -1381,6 +1385,8 @@ abstract public class Task implements Writable, 
Configurable {
         NetUtils.addStaticResolution(name, resolvedName);
       }
     }
+
+    uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8b67e4b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 0000000..6bf0601
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ExitUtil.ExitException;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestTask {
+  @Mock
+  private TaskUmbilicalProtocol umbilical;
+
+  private Task task;
+
+  @Before
+  public void setup() {
+    task = new StubTask();
+    ExitUtil.disableSystemExit();
+  }
+
+  @Test
+  public void testStatusUpdateDoesNotExitInUberMode() throws Exception {
+    setupTest(true);
+
+    task.statusUpdate(umbilical);
+  }
+
+  @Test(expected = ExitException.class)
+  public void testStatusUpdateExitsInNonUberMode() throws Exception {
+    setupTest(false);
+
+    task.statusUpdate(umbilical);
+  }
+
+  private void setupTest(boolean uberized)
+      throws IOException, InterruptedException {
+    Configuration conf = new Configuration(false);
+    conf.setBoolean("mapreduce.task.uberized", uberized);
+    task.setConf(conf);
+    // (false, true) to avoid possible infinite loop
+    when(umbilical.statusUpdate(any(TaskAttemptID.class),
+        any(TaskStatus.class))).thenReturn(false, true);
+  }
+
+  public class StubTask extends Task {
+    @Override
+    public void run(JobConf job, TaskUmbilicalProtocol umbilical)
+        throws IOException, ClassNotFoundException, InterruptedException {
+      // nop
+    }
+
+    @Override
+    public boolean isMapTask() {
+      return false;
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

Reply via email to