[23/50] hadoop git commit: MAPREDUCE-6797. Job history server scans can become blocked on a single, slow entry. Contributed by Prabhu Joseph

2016-11-16 Thread kasha 
MAPREDUCE-6797. Job history server scans can become blocked on a single, slow 
entry. Contributed by Prabhu Joseph


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99c2bbd3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99c2bbd3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99c2bbd3

Branch: refs/heads/YARN-4752
Commit: 99c2bbd337942e4bc7b246a88dff53f98e530651
Parents: ff0b99e
Author: Jason Lowe 
Authored: Mon Nov 14 20:20:50 2016 +
Committer: Jason Lowe 
Committed: Mon Nov 14 20:20:50 2016 +

--
 .../mapreduce/v2/hs/HistoryFileManager.java | 48 ++--
 1 file changed, 23 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c2bbd3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 0f09df2..bd8d9c2 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -231,38 +231,36 @@ public class HistoryFileManager extends AbstractService {
 JobId firstMoveFailedKey = null;
 int moveFailedCount = 0;
 
-while(cache.size() > maxSize && keys.hasNext()) {
+while (cache.size() > maxSize && keys.hasNext()) {
   JobId key = keys.next();
   HistoryFileInfo firstValue = cache.get(key);
-  if(firstValue != null) {
-synchronized(firstValue) {
-  if (firstValue.isMovePending()) {
-if(firstValue.didMoveFail() &&
-firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
-  cache.remove(key);
-  //Now lets try to delete it
-  try {
-firstValue.delete();
-  } catch (IOException e) {
-LOG.error("Error while trying to delete history files" +
-   " that could not be moved to done.", e);
+  if (firstValue != null) {
+if (firstValue.isMovePending()) {
+  if (firstValue.didMoveFail() &&
+  firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
+cache.remove(key);
+// Now lets try to delete it
+try {
+  firstValue.delete();
+} catch (IOException e) {
+  LOG.error("Error while trying to delete history files" +
+  " that could not be moved to done.", e);
+}
+  } else {
+if (firstValue.didMoveFail()) {
+  if (moveFailedCount == 0) {
+firstMoveFailedKey = key;
   }
+  moveFailedCount += 1;
 } else {
-  if (firstValue.didMoveFail()) {
-if (moveFailedCount == 0) {
-  firstMoveFailedKey = key;
-}
-moveFailedCount += 1;
-  } else {
-if (inIntermediateCount == 0) {
-  firstInIntermediateKey = key;
-}
-inIntermediateCount += 1;
+  if (inIntermediateCount == 0) {
+firstInIntermediateKey = key;
   }
+  inIntermediateCount += 1;
 }
-  } else {
-cache.remove(key);
   }
+} else {
+  cache.remove(key);
 }
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[28/50] [abbrv] hadoop git commit: MAPREDUCE-6797. Job history server scans can become blocked on a single, slow entry. Contributed by Prabhu Joseph

2016-11-16 Thread vvasudev 
MAPREDUCE-6797. Job history server scans can become blocked on a single, slow 
entry. Contributed by Prabhu Joseph


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99c2bbd3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99c2bbd3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99c2bbd3

Branch: refs/heads/YARN-3926
Commit: 99c2bbd337942e4bc7b246a88dff53f98e530651
Parents: ff0b99e
Author: Jason Lowe 
Authored: Mon Nov 14 20:20:50 2016 +
Committer: Jason Lowe 
Committed: Mon Nov 14 20:20:50 2016 +

--
 .../mapreduce/v2/hs/HistoryFileManager.java | 48 ++--
 1 file changed, 23 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c2bbd3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 0f09df2..bd8d9c2 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -231,38 +231,36 @@ public class HistoryFileManager extends AbstractService {
 JobId firstMoveFailedKey = null;
 int moveFailedCount = 0;
 
-while(cache.size() > maxSize && keys.hasNext()) {
+while (cache.size() > maxSize && keys.hasNext()) {
   JobId key = keys.next();
   HistoryFileInfo firstValue = cache.get(key);
-  if(firstValue != null) {
-synchronized(firstValue) {
-  if (firstValue.isMovePending()) {
-if(firstValue.didMoveFail() &&
-firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
-  cache.remove(key);
-  //Now lets try to delete it
-  try {
-firstValue.delete();
-  } catch (IOException e) {
-LOG.error("Error while trying to delete history files" +
-   " that could not be moved to done.", e);
+  if (firstValue != null) {
+if (firstValue.isMovePending()) {
+  if (firstValue.didMoveFail() &&
+  firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
+cache.remove(key);
+// Now lets try to delete it
+try {
+  firstValue.delete();
+} catch (IOException e) {
+  LOG.error("Error while trying to delete history files" +
+  " that could not be moved to done.", e);
+}
+  } else {
+if (firstValue.didMoveFail()) {
+  if (moveFailedCount == 0) {
+firstMoveFailedKey = key;
   }
+  moveFailedCount += 1;
 } else {
-  if (firstValue.didMoveFail()) {
-if (moveFailedCount == 0) {
-  firstMoveFailedKey = key;
-}
-moveFailedCount += 1;
-  } else {
-if (inIntermediateCount == 0) {
-  firstInIntermediateKey = key;
-}
-inIntermediateCount += 1;
+  if (inIntermediateCount == 0) {
+firstInIntermediateKey = key;
   }
+  inIntermediateCount += 1;
 }
-  } else {
-cache.remove(key);
   }
+} else {
+  cache.remove(key);
 }
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6797. Job history server scans can become blocked on a single, slow entry. Contributed by Prabhu Joseph (cherry picked from commit 99c2bbd337942e4bc7b246a88dff53f98e530651

2016-11-14 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 4dc40fe28 -> 5b1b825ac


MAPREDUCE-6797. Job history server scans can become blocked on a single, slow 
entry. Contributed by Prabhu Joseph
(cherry picked from commit 99c2bbd337942e4bc7b246a88dff53f98e530651)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b1b825a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b1b825a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b1b825a

Branch: refs/heads/branch-2.8
Commit: 5b1b825ac8dbac32699de5a9486960dca30ddb75
Parents: 4dc40fe
Author: Jason Lowe 
Authored: Mon Nov 14 20:20:50 2016 +
Committer: Jason Lowe 
Committed: Mon Nov 14 20:31:38 2016 +

--
 .../mapreduce/v2/hs/HistoryFileManager.java | 48 ++--
 1 file changed, 23 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b1b825a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index e8c84da..7994f8a 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -227,38 +227,36 @@ public class HistoryFileManager extends AbstractService {
 JobId firstMoveFailedKey = null;
 int moveFailedCount = 0;
 
-while(cache.size() > maxSize && keys.hasNext()) {
+while (cache.size() > maxSize && keys.hasNext()) {
   JobId key = keys.next();
   HistoryFileInfo firstValue = cache.get(key);
-  if(firstValue != null) {
-synchronized(firstValue) {
-  if (firstValue.isMovePending()) {
-if(firstValue.didMoveFail() &&
-firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
-  cache.remove(key);
-  //Now lets try to delete it
-  try {
-firstValue.delete();
-  } catch (IOException e) {
-LOG.error("Error while trying to delete history files" +
-   " that could not be moved to done.", e);
+  if (firstValue != null) {
+if (firstValue.isMovePending()) {
+  if (firstValue.didMoveFail() &&
+  firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
+cache.remove(key);
+// Now lets try to delete it
+try {
+  firstValue.delete();
+} catch (IOException e) {
+  LOG.error("Error while trying to delete history files" +
+  " that could not be moved to done.", e);
+}
+  } else {
+if (firstValue.didMoveFail()) {
+  if (moveFailedCount == 0) {
+firstMoveFailedKey = key;
   }
+  moveFailedCount += 1;
 } else {
-  if (firstValue.didMoveFail()) {
-if (moveFailedCount == 0) {
-  firstMoveFailedKey = key;
-}
-moveFailedCount += 1;
-  } else {
-if (inIntermediateCount == 0) {
-  firstInIntermediateKey = key;
-}
-inIntermediateCount += 1;
+  if (inIntermediateCount == 0) {
+firstInIntermediateKey = key;
   }
+  inIntermediateCount += 1;
 }
-  } else {
-cache.remove(key);
   }
+} else {
+  cache.remove(key);
 }
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6797. Job history server scans can become blocked on a single, slow entry. Contributed by Prabhu Joseph (cherry picked from commit 99c2bbd337942e4bc7b246a88dff53f98e530651

2016-11-14 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 399abe7c7 -> 5ffb26c2a


MAPREDUCE-6797. Job history server scans can become blocked on a single, slow 
entry. Contributed by Prabhu Joseph
(cherry picked from commit 99c2bbd337942e4bc7b246a88dff53f98e530651)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ffb26c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ffb26c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ffb26c2

Branch: refs/heads/branch-2
Commit: 5ffb26c2a9e03fb57057462897dff7dc65887034
Parents: 399abe7
Author: Jason Lowe 
Authored: Mon Nov 14 20:20:50 2016 +
Committer: Jason Lowe 
Committed: Mon Nov 14 20:30:36 2016 +

--
 .../mapreduce/v2/hs/HistoryFileManager.java | 48 ++--
 1 file changed, 23 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ffb26c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 69faba9..0789c35 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -231,38 +231,36 @@ public class HistoryFileManager extends AbstractService {
 JobId firstMoveFailedKey = null;
 int moveFailedCount = 0;
 
-while(cache.size() > maxSize && keys.hasNext()) {
+while (cache.size() > maxSize && keys.hasNext()) {
   JobId key = keys.next();
   HistoryFileInfo firstValue = cache.get(key);
-  if(firstValue != null) {
-synchronized(firstValue) {
-  if (firstValue.isMovePending()) {
-if(firstValue.didMoveFail() &&
-firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
-  cache.remove(key);
-  //Now lets try to delete it
-  try {
-firstValue.delete();
-  } catch (IOException e) {
-LOG.error("Error while trying to delete history files" +
-   " that could not be moved to done.", e);
+  if (firstValue != null) {
+if (firstValue.isMovePending()) {
+  if (firstValue.didMoveFail() &&
+  firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
+cache.remove(key);
+// Now lets try to delete it
+try {
+  firstValue.delete();
+} catch (IOException e) {
+  LOG.error("Error while trying to delete history files" +
+  " that could not be moved to done.", e);
+}
+  } else {
+if (firstValue.didMoveFail()) {
+  if (moveFailedCount == 0) {
+firstMoveFailedKey = key;
   }
+  moveFailedCount += 1;
 } else {
-  if (firstValue.didMoveFail()) {
-if (moveFailedCount == 0) {
-  firstMoveFailedKey = key;
-}
-moveFailedCount += 1;
-  } else {
-if (inIntermediateCount == 0) {
-  firstInIntermediateKey = key;
-}
-inIntermediateCount += 1;
+  if (inIntermediateCount == 0) {
+firstInIntermediateKey = key;
   }
+  inIntermediateCount += 1;
 }
-  } else {
-cache.remove(key);
   }
+} else {
+  cache.remove(key);
 }
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6797. Job history server scans can become blocked on a single, slow entry. Contributed by Prabhu Joseph

2016-11-14 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ff0b99eaf -> 99c2bbd33


MAPREDUCE-6797. Job history server scans can become blocked on a single, slow 
entry. Contributed by Prabhu Joseph


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99c2bbd3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99c2bbd3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99c2bbd3

Branch: refs/heads/trunk
Commit: 99c2bbd337942e4bc7b246a88dff53f98e530651
Parents: ff0b99e
Author: Jason Lowe 
Authored: Mon Nov 14 20:20:50 2016 +
Committer: Jason Lowe 
Committed: Mon Nov 14 20:20:50 2016 +

--
 .../mapreduce/v2/hs/HistoryFileManager.java | 48 ++--
 1 file changed, 23 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c2bbd3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 0f09df2..bd8d9c2 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -231,38 +231,36 @@ public class HistoryFileManager extends AbstractService {
 JobId firstMoveFailedKey = null;
 int moveFailedCount = 0;
 
-while(cache.size() > maxSize && keys.hasNext()) {
+while (cache.size() > maxSize && keys.hasNext()) {
   JobId key = keys.next();
   HistoryFileInfo firstValue = cache.get(key);
-  if(firstValue != null) {
-synchronized(firstValue) {
-  if (firstValue.isMovePending()) {
-if(firstValue.didMoveFail() &&
-firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
-  cache.remove(key);
-  //Now lets try to delete it
-  try {
-firstValue.delete();
-  } catch (IOException e) {
-LOG.error("Error while trying to delete history files" +
-   " that could not be moved to done.", e);
+  if (firstValue != null) {
+if (firstValue.isMovePending()) {
+  if (firstValue.didMoveFail() &&
+  firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
+cache.remove(key);
+// Now lets try to delete it
+try {
+  firstValue.delete();
+} catch (IOException e) {
+  LOG.error("Error while trying to delete history files" +
+  " that could not be moved to done.", e);
+}
+  } else {
+if (firstValue.didMoveFail()) {
+  if (moveFailedCount == 0) {
+firstMoveFailedKey = key;
   }
+  moveFailedCount += 1;
 } else {
-  if (firstValue.didMoveFail()) {
-if (moveFailedCount == 0) {
-  firstMoveFailedKey = key;
-}
-moveFailedCount += 1;
-  } else {
-if (inIntermediateCount == 0) {
-  firstInIntermediateKey = key;
-}
-inIntermediateCount += 1;
+  if (inIntermediateCount == 0) {
+firstInIntermediateKey = key;
   }
+  inIntermediateCount += 1;
 }
-  } else {
-cache.remove(key);
   }
+} else {
+  cache.remove(key);
 }
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org