[09/50] [abbrv] hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla

2017-06-06 Thread liuml07 
YARN-6641. Non-public resource localization on a bad disk causes subsequent 
containers failure. Contributed by Kuhu Shukla


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aea42930
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aea42930
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aea42930

Branch: refs/heads/HADOOP-13345
Commit: aea42930bbb9566ea6988f684dbd72a72a2bdadf
Parents: 47474ff
Author: Jason Lowe 
Authored: Fri May 26 09:37:56 2017 -0500
Committer: Jason Lowe 
Committed: Fri May 26 09:37:56 2017 -0500

--
 .../localizer/LocalResourcesTrackerImpl.java| 13 ++--
 .../localizer/ResourceLocalizationService.java  | 10 +--
 .../TestLocalResourcesTrackerImpl.java  |  8 +--
 .../TestResourceLocalizationService.java| 71 
 4 files changed, 86 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aea42930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 
   public LocalResourcesTrackerImpl(String user, ApplicationId appId,
   Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
-  Configuration conf, NMStateStoreService stateStore) {
-this(user, appId, dispatcher,
-new ConcurrentHashMap(),
-useLocalCacheDirectoryManager, conf, stateStore, null);
-  }
-
-  public LocalResourcesTrackerImpl(String user, ApplicationId appId,
-  Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
   Configuration conf, NMStateStoreService stateStore,
   LocalDirsHandlerService dirHandler) {
 this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 }
 return mgr;
   }
+
+  @VisibleForTesting
+  LocalDirsHandlerService getDirsHandler() {
+return dirsHandler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aea42930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 28fb53c..663bad7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -306,7 +306,7 @@ public class ResourceLocalizationService extends 
CompositeService
   trackerState = userResources.getPrivateTrackerState();
   if (!trackerState.isEmpty()) {
 LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-null, dispatcher, true, super.getConfig(), stateStore);
+null, dispatcher, true, super.getConfig(), stateStore, 
dirsHandler);
 LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user,
 tracker);
 if (oldTracker != null) {
@@ -322,7 +322,8 @@

[02/50] [abbrv] hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla

2017-06-05 Thread xgong 
YARN-6641. Non-public resource localization on a bad disk causes subsequent 
containers failure. Contributed by Kuhu Shukla


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aea42930
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aea42930
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aea42930

Branch: refs/heads/YARN-5734
Commit: aea42930bbb9566ea6988f684dbd72a72a2bdadf
Parents: 47474ff
Author: Jason Lowe 
Authored: Fri May 26 09:37:56 2017 -0500
Committer: Jason Lowe 
Committed: Fri May 26 09:37:56 2017 -0500

--
 .../localizer/LocalResourcesTrackerImpl.java| 13 ++--
 .../localizer/ResourceLocalizationService.java  | 10 +--
 .../TestLocalResourcesTrackerImpl.java  |  8 +--
 .../TestResourceLocalizationService.java| 71 
 4 files changed, 86 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aea42930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 
   public LocalResourcesTrackerImpl(String user, ApplicationId appId,
   Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
-  Configuration conf, NMStateStoreService stateStore) {
-this(user, appId, dispatcher,
-new ConcurrentHashMap(),
-useLocalCacheDirectoryManager, conf, stateStore, null);
-  }
-
-  public LocalResourcesTrackerImpl(String user, ApplicationId appId,
-  Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
   Configuration conf, NMStateStoreService stateStore,
   LocalDirsHandlerService dirHandler) {
 this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 }
 return mgr;
   }
+
+  @VisibleForTesting
+  LocalDirsHandlerService getDirsHandler() {
+return dirsHandler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aea42930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 28fb53c..663bad7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -306,7 +306,7 @@ public class ResourceLocalizationService extends 
CompositeService
   trackerState = userResources.getPrivateTrackerState();
   if (!trackerState.isEmpty()) {
 LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-null, dispatcher, true, super.getConfig(), stateStore);
+null, dispatcher, true, super.getConfig(), stateStore, 
dirsHandler);
 LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user,
 tracker);
 if (oldTracker != null) {
@@ -322,7 +322,8 @@

[07/50] [abbrv] hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla

2017-06-05 Thread xgong 
YARN-6641. Non-public resource localization on a bad disk causes subsequent 
containers failure. Contributed by Kuhu Shukla


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47710c22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47710c22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47710c22

Branch: refs/heads/YARN-5734
Commit: 47710c22fc3a70e3fc1504f4235bd8ca5e1b1671
Parents: 65cbe43
Author: Jason Lowe 
Authored: Fri May 26 09:37:56 2017 -0500
Committer: Xuan 
Committed: Mon Jun 5 13:29:22 2017 -0700

--
 .../localizer/LocalResourcesTrackerImpl.java| 13 ++--
 .../localizer/ResourceLocalizationService.java  | 10 +--
 .../TestLocalResourcesTrackerImpl.java  |  8 +--
 .../TestResourceLocalizationService.java| 71 
 4 files changed, 86 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/47710c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 
   public LocalResourcesTrackerImpl(String user, ApplicationId appId,
   Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
-  Configuration conf, NMStateStoreService stateStore) {
-this(user, appId, dispatcher,
-new ConcurrentHashMap(),
-useLocalCacheDirectoryManager, conf, stateStore, null);
-  }
-
-  public LocalResourcesTrackerImpl(String user, ApplicationId appId,
-  Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
   Configuration conf, NMStateStoreService stateStore,
   LocalDirsHandlerService dirHandler) {
 this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 }
 return mgr;
   }
+
+  @VisibleForTesting
+  LocalDirsHandlerService getDirsHandler() {
+return dirsHandler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/47710c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 28fb53c..663bad7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -306,7 +306,7 @@ public class ResourceLocalizationService extends 
CompositeService
   trackerState = userResources.getPrivateTrackerState();
   if (!trackerState.isEmpty()) {
 LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-null, dispatcher, true, super.getConfig(), stateStore);
+null, dispatcher, true, super.getConfig(), stateStore, 
dirsHandler);
 LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user,
 tracker);
 if (oldTracker != null) {
@@ -322,7 +322,8 @@ public class

[28/50] [abbrv] hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla

2017-05-31 Thread haibochen 
YARN-6641. Non-public resource localization on a bad disk causes subsequent 
containers failure. Contributed by Kuhu Shukla


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aea42930
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aea42930
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aea42930

Branch: refs/heads/YARN-1011
Commit: aea42930bbb9566ea6988f684dbd72a72a2bdadf
Parents: 47474ff
Author: Jason Lowe 
Authored: Fri May 26 09:37:56 2017 -0500
Committer: Jason Lowe 
Committed: Fri May 26 09:37:56 2017 -0500

--
 .../localizer/LocalResourcesTrackerImpl.java| 13 ++--
 .../localizer/ResourceLocalizationService.java  | 10 +--
 .../TestLocalResourcesTrackerImpl.java  |  8 +--
 .../TestResourceLocalizationService.java| 71 
 4 files changed, 86 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aea42930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 
   public LocalResourcesTrackerImpl(String user, ApplicationId appId,
   Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
-  Configuration conf, NMStateStoreService stateStore) {
-this(user, appId, dispatcher,
-new ConcurrentHashMap(),
-useLocalCacheDirectoryManager, conf, stateStore, null);
-  }
-
-  public LocalResourcesTrackerImpl(String user, ApplicationId appId,
-  Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
   Configuration conf, NMStateStoreService stateStore,
   LocalDirsHandlerService dirHandler) {
 this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 }
 return mgr;
   }
+
+  @VisibleForTesting
+  LocalDirsHandlerService getDirsHandler() {
+return dirsHandler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aea42930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 28fb53c..663bad7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -306,7 +306,7 @@ public class ResourceLocalizationService extends 
CompositeService
   trackerState = userResources.getPrivateTrackerState();
   if (!trackerState.isEmpty()) {
 LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-null, dispatcher, true, super.getConfig(), stateStore);
+null, dispatcher, true, super.getConfig(), stateStore, 
dirsHandler);
 LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user,
 tracker);
 if (oldTracker != null) {
@@ -322,7 +322,8 @@

hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla

2017-05-26 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8.1 e2a817a9a -> 1d597defb


YARN-6641. Non-public resource localization on a bad disk causes subsequent 
containers failure. Contributed by Kuhu Shukla

(cherry picked from commit b89d59e21eb7a7b465f666ccfadc9927fd12d967)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d597def
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d597def
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d597def

Branch: refs/heads/branch-2.8.1
Commit: 1d597defb1a8bfcaabe8b7089a9fe1222318b0c5
Parents: e2a817a
Author: Jason Lowe 
Authored: Fri May 26 09:45:49 2017 -0500
Committer: Jason Lowe 
Committed: Fri May 26 09:46:44 2017 -0500

--
 .../localizer/LocalResourcesTrackerImpl.java| 13 ++--
 .../localizer/ResourceLocalizationService.java  | 10 +--
 .../TestLocalResourcesTrackerImpl.java  |  8 +--
 .../TestResourceLocalizationService.java| 71 
 4 files changed, 86 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d597def/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 
   public LocalResourcesTrackerImpl(String user, ApplicationId appId,
   Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
-  Configuration conf, NMStateStoreService stateStore) {
-this(user, appId, dispatcher,
-new ConcurrentHashMap(),
-useLocalCacheDirectoryManager, conf, stateStore, null);
-  }
-
-  public LocalResourcesTrackerImpl(String user, ApplicationId appId,
-  Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
   Configuration conf, NMStateStoreService stateStore,
   LocalDirsHandlerService dirHandler) {
 this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 }
 return mgr;
   }
+
+  @VisibleForTesting
+  LocalDirsHandlerService getDirsHandler() {
+return dirsHandler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d597def/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 61b6cdd..353831d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -293,7 +293,7 @@ public class ResourceLocalizationService extends 
CompositeService
   trackerState = userResources.getPrivateTrackerState();
   if (!trackerState.isEmpty()) {
 LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-null, dispatcher, true, super.getConfig(), stateStore);
+null, dispatcher, true, super.getConfig(), stateStore,

hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla

2017-05-26 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 bbef16b84 -> b89d59e21


YARN-6641. Non-public resource localization on a bad disk causes subsequent 
containers failure. Contributed by Kuhu Shukla


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b89d59e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b89d59e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b89d59e2

Branch: refs/heads/branch-2.8
Commit: b89d59e21eb7a7b465f666ccfadc9927fd12d967
Parents: bbef16b
Author: Jason Lowe 
Authored: Fri May 26 09:45:49 2017 -0500
Committer: Jason Lowe 
Committed: Fri May 26 09:45:49 2017 -0500

--
 .../localizer/LocalResourcesTrackerImpl.java| 13 ++--
 .../localizer/ResourceLocalizationService.java  | 10 +--
 .../TestLocalResourcesTrackerImpl.java  |  8 +--
 .../TestResourceLocalizationService.java| 71 
 4 files changed, 86 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89d59e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 
   public LocalResourcesTrackerImpl(String user, ApplicationId appId,
   Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
-  Configuration conf, NMStateStoreService stateStore) {
-this(user, appId, dispatcher,
-new ConcurrentHashMap(),
-useLocalCacheDirectoryManager, conf, stateStore, null);
-  }
-
-  public LocalResourcesTrackerImpl(String user, ApplicationId appId,
-  Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
   Configuration conf, NMStateStoreService stateStore,
   LocalDirsHandlerService dirHandler) {
 this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 }
 return mgr;
   }
+
+  @VisibleForTesting
+  LocalDirsHandlerService getDirsHandler() {
+return dirsHandler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89d59e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 61b6cdd..353831d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -293,7 +293,7 @@ public class ResourceLocalizationService extends 
CompositeService
   trackerState = userResources.getPrivateTrackerState();
   if (!trackerState.isEmpty()) {
 LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-null, dispatcher, true, super.getConfig(), stateStore);
+null, dispatcher, true, super.getConfig(), stateStore, 
dirsHandler);
 LocalResourcesTracker oldTracker =

hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla

2017-05-26 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 54971c419 -> 3c0eb9c41


YARN-6641. Non-public resource localization on a bad disk causes subsequent 
containers failure. Contributed by Kuhu Shukla

(cherry picked from commit aea42930bbb9566ea6988f684dbd72a72a2bdadf)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c0eb9c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c0eb9c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c0eb9c4

Branch: refs/heads/branch-2
Commit: 3c0eb9c415fc9c6b05d950c906f618fa1b41286f
Parents: 54971c4
Author: Jason Lowe 
Authored: Fri May 26 09:37:56 2017 -0500
Committer: Jason Lowe 
Committed: Fri May 26 09:39:22 2017 -0500

--
 .../localizer/LocalResourcesTrackerImpl.java| 13 ++--
 .../localizer/ResourceLocalizationService.java  | 10 +--
 .../TestLocalResourcesTrackerImpl.java  |  8 +--
 .../TestResourceLocalizationService.java| 71 
 4 files changed, 86 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c0eb9c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 
   public LocalResourcesTrackerImpl(String user, ApplicationId appId,
   Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
-  Configuration conf, NMStateStoreService stateStore) {
-this(user, appId, dispatcher,
-new ConcurrentHashMap(),
-useLocalCacheDirectoryManager, conf, stateStore, null);
-  }
-
-  public LocalResourcesTrackerImpl(String user, ApplicationId appId,
-  Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
   Configuration conf, NMStateStoreService stateStore,
   LocalDirsHandlerService dirHandler) {
 this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 }
 return mgr;
   }
+
+  @VisibleForTesting
+  LocalDirsHandlerService getDirsHandler() {
+return dirsHandler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c0eb9c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 28fb53c..663bad7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -306,7 +306,7 @@ public class ResourceLocalizationService extends 
CompositeService
   trackerState = userResources.getPrivateTrackerState();
   if (!trackerState.isEmpty()) {
 LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-null, dispatcher, true, super.getConfig(), stateStore);
+null, dispatcher, true, super.getConfig(), stateStore, 
dirsHandler);

hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla

2017-05-26 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 47474fffa -> aea42930b


YARN-6641. Non-public resource localization on a bad disk causes subsequent 
containers failure. Contributed by Kuhu Shukla


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aea42930
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aea42930
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aea42930

Branch: refs/heads/trunk
Commit: aea42930bbb9566ea6988f684dbd72a72a2bdadf
Parents: 47474ff
Author: Jason Lowe 
Authored: Fri May 26 09:37:56 2017 -0500
Committer: Jason Lowe 
Committed: Fri May 26 09:37:56 2017 -0500

--
 .../localizer/LocalResourcesTrackerImpl.java| 13 ++--
 .../localizer/ResourceLocalizationService.java  | 10 +--
 .../TestLocalResourcesTrackerImpl.java  |  8 +--
 .../TestResourceLocalizationService.java| 71 
 4 files changed, 86 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aea42930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 
   public LocalResourcesTrackerImpl(String user, ApplicationId appId,
   Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
-  Configuration conf, NMStateStoreService stateStore) {
-this(user, appId, dispatcher,
-new ConcurrentHashMap(),
-useLocalCacheDirectoryManager, conf, stateStore, null);
-  }
-
-  public LocalResourcesTrackerImpl(String user, ApplicationId appId,
-  Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
   Configuration conf, NMStateStoreService stateStore,
   LocalDirsHandlerService dirHandler) {
 this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements 
LocalResourcesTracker {
 }
 return mgr;
   }
+
+  @VisibleForTesting
+  LocalDirsHandlerService getDirsHandler() {
+return dirsHandler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aea42930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 28fb53c..663bad7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -306,7 +306,7 @@ public class ResourceLocalizationService extends 
CompositeService
   trackerState = userResources.getPrivateTrackerState();
   if (!trackerState.isEmpty()) {
 LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-null, dispatcher, true, super.getConfig(), stateStore);
+null, dispatcher, true, super.getConfig(), stateStore, 
dirsHandler);
 LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user,