[24/50] [abbrv] hadoop git commit: MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. Contributed by Peter Bacsko

2018-02-15 Thread aengineer 
MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87e2570a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87e2570a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87e2570a

Branch: refs/heads/HDFS-7240
Commit: 87e2570a1419d3616de2de3b553108ad1a8af425
Parents: 35c1735
Author: Jason Lowe 
Authored: Mon Feb 12 13:21:09 2018 -0600
Committer: Jason Lowe 
Committed: Mon Feb 12 13:21:09 2018 -0600

--
 .../java/org/apache/hadoop/mapred/Task.java | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 89 
 2 files changed, 100 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/87e2570a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 5b98b35..d83a6b0 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -200,6 +200,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
   final AtomicBoolean mustPreempt = new AtomicBoolean(false);
+  private boolean uberized = false;
 
   
   // Constructors
@@ -855,9 +856,6 @@ abstract public class Task implements Writable, 
Configurable {
   long taskProgressInterval = MRJobConfUtil.
   getTaskProgressReportInterval(conf);
 
-  boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-  false);
-
   while (!taskDone.get()) {
 synchronized (lock) {
   done = false;
@@ -1301,11 +1299,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
 int retries = MAX_RETRIES;
+
 while (true) {
   try {
 if (!umbilical.statusUpdate(getTaskID(), taskStatus).getTaskFound()) {
-  LOG.warn("Parent died.  Exiting "+taskId);
-  System.exit(66);
+  if (uberized) {
+LOG.warn("Task no longer available: " + taskId);
+break;
+  } else {
+LOG.warn("Parent died.  Exiting " + taskId);
+ExitUtil.terminate(66);
+  }
 }
 taskStatus.clearStatus();
 return;
@@ -1518,6 +1522,8 @@ abstract public class Task implements Writable, 
Configurable {
 NetUtils.addStaticResolution(name, resolvedName);
   }
 }
+
+uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87e2570a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 000..500229c
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ *

hadoop git commit: MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. Contributed by Peter Bacsko

2018-02-12 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 055d5e0b8 -> bc2d4ba50


MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko

(cherry picked from commit 87e2570a1419d3616de2de3b553108ad1a8af425)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc2d4ba5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc2d4ba5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc2d4ba5

Branch: refs/heads/branch-3.0
Commit: bc2d4ba5075fa8eebbb74a51da3aa1c5df7d180e
Parents: 055d5e0
Author: Jason Lowe 
Authored: Mon Feb 12 13:21:09 2018 -0600
Committer: Jason Lowe 
Committed: Mon Feb 12 14:49:35 2018 -0600

--
 .../java/org/apache/hadoop/mapred/Task.java | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 89 
 2 files changed, 100 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2d4ba5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 9059258..bcc0351 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -199,6 +199,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
   final AtomicBoolean mustPreempt = new AtomicBoolean(false);
+  private boolean uberized = false;
 
   
   // Constructors
@@ -790,9 +791,6 @@ abstract public class Task implements Writable, 
Configurable {
   long taskProgressInterval = MRJobConfUtil.
   getTaskProgressReportInterval(conf);
 
-  boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-  false);
-
   while (!taskDone.get()) {
 synchronized (lock) {
   done = false;
@@ -1220,11 +1218,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
 int retries = MAX_RETRIES;
+
 while (true) {
   try {
 if (!umbilical.statusUpdate(getTaskID(), taskStatus).getTaskFound()) {
-  LOG.warn("Parent died.  Exiting "+taskId);
-  System.exit(66);
+  if (uberized) {
+LOG.warn("Task no longer available: " + taskId);
+break;
+  } else {
+LOG.warn("Parent died.  Exiting " + taskId);
+ExitUtil.terminate(66);
+  }
 }
 taskStatus.clearStatus();
 return;
@@ -1437,6 +1441,8 @@ abstract public class Task implements Writable, 
Configurable {
 NetUtils.addStaticResolution(name, resolvedName);
   }
 }
+
+uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2d4ba5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 000..500229c
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+

hadoop git commit: MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. Contributed by Peter Bacsko

2018-02-12 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 77a933a45 -> 486fa889d


MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko

(cherry picked from commit 08feac4c83939f60289f8540d72ffcabb7852e2a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/486fa889
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/486fa889
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/486fa889

Branch: refs/heads/branch-2.8
Commit: 486fa889d95c8da77236240ed733df54c5fb4026
Parents: 77a933a
Author: Jason Lowe 
Authored: Mon Feb 12 14:30:42 2018 -0600
Committer: Jason Lowe 
Committed: Mon Feb 12 14:37:59 2018 -0600

--
 .../java/org/apache/hadoop/mapred/Task.java | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 84 
 2 files changed, 95 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/486fa889/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 57d05ba..c8c092c 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -195,6 +195,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey tokenSecret;
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
+  private boolean uberized = false;
 
   
   // Constructors
@@ -785,9 +786,6 @@ abstract public class Task implements Writable, 
Configurable {
   long taskProgressInterval = MRJobConfUtil.
   getTaskProgressReportInterval(conf);
 
-  boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-  false);
-
   while (!taskDone.get()) {
 synchronized (lock) {
   done = false;
@@ -1176,11 +1174,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
 int retries = MAX_RETRIES;
+
 while (true) {
   try {
 if (!umbilical.statusUpdate(getTaskID(), taskStatus)) {
-  LOG.warn("Parent died.  Exiting "+taskId);
-  System.exit(66);
+  if (uberized) {
+LOG.warn("Task no longer available: " + taskId);
+break;
+  } else {
+LOG.warn("Parent died.  Exiting " + taskId);
+ExitUtil.terminate(66);
+  }
 }
 taskStatus.clearStatus();
 return;
@@ -1393,6 +1397,8 @@ abstract public class Task implements Writable, 
Configurable {
 NetUtils.addStaticResolution(name, resolvedName);
   }
 }
+
+uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/486fa889/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 000..6bf0601
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY

hadoop git commit: MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. Contributed by Peter Bacsko

2018-02-12 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 ed512645b -> b8b67e4b8


MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko

(cherry picked from commit 08feac4c83939f60289f8540d72ffcabb7852e2a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8b67e4b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8b67e4b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8b67e4b

Branch: refs/heads/branch-2.9
Commit: b8b67e4b8da02dd99ab222c9548a5959acfd4df8
Parents: ed51264
Author: Jason Lowe 
Authored: Mon Feb 12 14:30:42 2018 -0600
Committer: Jason Lowe 
Committed: Mon Feb 12 14:33:32 2018 -0600

--
 .../java/org/apache/hadoop/mapred/Task.java | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 84 
 2 files changed, 95 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8b67e4b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 169164f..a9ac184 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -195,6 +195,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey tokenSecret;
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
+  private boolean uberized = false;
 
   
   // Constructors
@@ -785,9 +786,6 @@ abstract public class Task implements Writable, 
Configurable {
   long taskProgressInterval = MRJobConfUtil.
   getTaskProgressReportInterval(conf);
 
-  boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-  false);
-
   while (!taskDone.get()) {
 synchronized (lock) {
   done = false;
@@ -1164,11 +1162,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
 int retries = MAX_RETRIES;
+
 while (true) {
   try {
 if (!umbilical.statusUpdate(getTaskID(), taskStatus)) {
-  LOG.warn("Parent died.  Exiting "+taskId);
-  System.exit(66);
+  if (uberized) {
+LOG.warn("Task no longer available: " + taskId);
+break;
+  } else {
+LOG.warn("Parent died.  Exiting " + taskId);
+ExitUtil.terminate(66);
+  }
 }
 taskStatus.clearStatus();
 return;
@@ -1381,6 +1385,8 @@ abstract public class Task implements Writable, 
Configurable {
 NetUtils.addStaticResolution(name, resolvedName);
   }
 }
+
+uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8b67e4b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 000..6bf0601
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY

hadoop git commit: MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. Contributed by Peter Bacsko

2018-02-12 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 24af0bc14 -> 08feac4c8


MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08feac4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08feac4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08feac4c

Branch: refs/heads/branch-2
Commit: 08feac4c83939f60289f8540d72ffcabb7852e2a
Parents: 24af0bc
Author: Jason Lowe 
Authored: Mon Feb 12 14:30:42 2018 -0600
Committer: Jason Lowe 
Committed: Mon Feb 12 14:30:42 2018 -0600

--
 .../java/org/apache/hadoop/mapred/Task.java | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 84 
 2 files changed, 95 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/08feac4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index e277325..a3007ec 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -195,6 +195,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey tokenSecret;
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
+  private boolean uberized = false;
 
   
   // Constructors
@@ -785,9 +786,6 @@ abstract public class Task implements Writable, 
Configurable {
   long taskProgressInterval = MRJobConfUtil.
   getTaskProgressReportInterval(conf);
 
-  boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-  false);
-
   while (!taskDone.get()) {
 synchronized (lock) {
   done = false;
@@ -1176,11 +1174,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
 int retries = MAX_RETRIES;
+
 while (true) {
   try {
 if (!umbilical.statusUpdate(getTaskID(), taskStatus)) {
-  LOG.warn("Parent died.  Exiting "+taskId);
-  System.exit(66);
+  if (uberized) {
+LOG.warn("Task no longer available: " + taskId);
+break;
+  } else {
+LOG.warn("Parent died.  Exiting " + taskId);
+ExitUtil.terminate(66);
+  }
 }
 taskStatus.clearStatus();
 return;
@@ -1393,6 +1397,8 @@ abstract public class Task implements Writable, 
Configurable {
 NetUtils.addStaticResolution(name, resolvedName);
   }
 }
+
+uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/08feac4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 000..6bf0601
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific

hadoop git commit: MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. Contributed by Peter Bacsko

2018-02-12 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0.1 e716b4359 -> 153039663


MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko

(cherry picked from commit 87e2570a1419d3616de2de3b553108ad1a8af425)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15303966
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15303966
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15303966

Branch: refs/heads/branch-3.0.1
Commit: 1530396639a612ebb5b4066543cfd7d9ec693de3
Parents: e716b43
Author: Jason Lowe 
Authored: Mon Feb 12 13:21:09 2018 -0600
Committer: Jason Lowe 
Committed: Mon Feb 12 13:30:51 2018 -0600

--
 .../java/org/apache/hadoop/mapred/Task.java | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 89 
 2 files changed, 100 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/15303966/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 9059258..bcc0351 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -199,6 +199,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
   final AtomicBoolean mustPreempt = new AtomicBoolean(false);
+  private boolean uberized = false;
 
   
   // Constructors
@@ -790,9 +791,6 @@ abstract public class Task implements Writable, 
Configurable {
   long taskProgressInterval = MRJobConfUtil.
   getTaskProgressReportInterval(conf);
 
-  boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-  false);
-
   while (!taskDone.get()) {
 synchronized (lock) {
   done = false;
@@ -1220,11 +1218,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
 int retries = MAX_RETRIES;
+
 while (true) {
   try {
 if (!umbilical.statusUpdate(getTaskID(), taskStatus).getTaskFound()) {
-  LOG.warn("Parent died.  Exiting "+taskId);
-  System.exit(66);
+  if (uberized) {
+LOG.warn("Task no longer available: " + taskId);
+break;
+  } else {
+LOG.warn("Parent died.  Exiting " + taskId);
+ExitUtil.terminate(66);
+  }
 }
 taskStatus.clearStatus();
 return;
@@ -1437,6 +1441,8 @@ abstract public class Task implements Writable, 
Configurable {
 NetUtils.addStaticResolution(name, resolvedName);
   }
 }
+
+uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/15303966/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 000..500229c
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS"

hadoop git commit: MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. Contributed by Peter Bacsko

2018-02-12 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 fb088564a -> 600e91946


MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko

(cherry picked from commit 87e2570a1419d3616de2de3b553108ad1a8af425)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/600e9194
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/600e9194
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/600e9194

Branch: refs/heads/branch-3.1
Commit: 600e9194691f8af53b9cf077afc70f9952c0cd68
Parents: fb08856
Author: Jason Lowe 
Authored: Mon Feb 12 13:21:09 2018 -0600
Committer: Jason Lowe 
Committed: Mon Feb 12 13:26:33 2018 -0600

--
 .../java/org/apache/hadoop/mapred/Task.java | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 89 
 2 files changed, 100 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/600e9194/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 5b98b35..d83a6b0 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -200,6 +200,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
   final AtomicBoolean mustPreempt = new AtomicBoolean(false);
+  private boolean uberized = false;
 
   
   // Constructors
@@ -855,9 +856,6 @@ abstract public class Task implements Writable, 
Configurable {
   long taskProgressInterval = MRJobConfUtil.
   getTaskProgressReportInterval(conf);
 
-  boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-  false);
-
   while (!taskDone.get()) {
 synchronized (lock) {
   done = false;
@@ -1301,11 +1299,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
 int retries = MAX_RETRIES;
+
 while (true) {
   try {
 if (!umbilical.statusUpdate(getTaskID(), taskStatus).getTaskFound()) {
-  LOG.warn("Parent died.  Exiting "+taskId);
-  System.exit(66);
+  if (uberized) {
+LOG.warn("Task no longer available: " + taskId);
+break;
+  } else {
+LOG.warn("Parent died.  Exiting " + taskId);
+ExitUtil.terminate(66);
+  }
 }
 taskStatus.clearStatus();
 return;
@@ -1518,6 +1522,8 @@ abstract public class Task implements Writable, 
Configurable {
 NetUtils.addStaticResolution(name, resolvedName);
   }
 }
+
+uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/600e9194/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 000..500229c
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+

hadoop git commit: MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. Contributed by Peter Bacsko

2018-02-12 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 35c17351c -> 87e2570a1


MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. 
Contributed by Peter Bacsko


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87e2570a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87e2570a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87e2570a

Branch: refs/heads/trunk
Commit: 87e2570a1419d3616de2de3b553108ad1a8af425
Parents: 35c1735
Author: Jason Lowe 
Authored: Mon Feb 12 13:21:09 2018 -0600
Committer: Jason Lowe 
Committed: Mon Feb 12 13:21:09 2018 -0600

--
 .../java/org/apache/hadoop/mapred/Task.java | 16 ++--
 .../java/org/apache/hadoop/mapred/TestTask.java | 89 
 2 files changed, 100 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/87e2570a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 5b98b35..d83a6b0 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -200,6 +200,7 @@ abstract public class Task implements Writable, 
Configurable {
   protected SecretKey shuffleSecret;
   protected GcTimeUpdater gcUpdater;
   final AtomicBoolean mustPreempt = new AtomicBoolean(false);
+  private boolean uberized = false;
 
   
   // Constructors
@@ -855,9 +856,6 @@ abstract public class Task implements Writable, 
Configurable {
   long taskProgressInterval = MRJobConfUtil.
   getTaskProgressReportInterval(conf);
 
-  boolean uberized = conf.getBoolean("mapreduce.task.uberized",
-  false);
-
   while (!taskDone.get()) {
 synchronized (lock) {
   done = false;
@@ -1301,11 +1299,17 @@ abstract public class Task implements Writable, 
Configurable {
   public void statusUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
 int retries = MAX_RETRIES;
+
 while (true) {
   try {
 if (!umbilical.statusUpdate(getTaskID(), taskStatus).getTaskFound()) {
-  LOG.warn("Parent died.  Exiting "+taskId);
-  System.exit(66);
+  if (uberized) {
+LOG.warn("Task no longer available: " + taskId);
+break;
+  } else {
+LOG.warn("Parent died.  Exiting " + taskId);
+ExitUtil.terminate(66);
+  }
 }
 taskStatus.clearStatus();
 return;
@@ -1518,6 +1522,8 @@ abstract public class Task implements Writable, 
Configurable {
 NetUtils.addStaticResolution(name, resolvedName);
   }
 }
+
+uberized = conf.getBoolean("mapreduce.task.uberized", false);
   }
 
   public Configuration getConf() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87e2570a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
new file mode 100644
index 000..500229c
--- /dev/null
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *