Repository: hadoop
Updated Branches:
  refs/heads/trunk cc3600aab -> a2cdffb95


MAPREDUCE-7094. LocalDistributedCacheManager leaves classloaders open, which 
leaks FDs. Contributed by Adam Szita.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2cdffb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2cdffb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2cdffb9

Branch: refs/heads/trunk
Commit: a2cdffb95acbcb3625ee72ebc8aeb8bf17fa4bc7
Parents: cc3600a
Author: Miklos Szegedi <szege...@apache.org>
Authored: Thu May 17 10:13:43 2018 -0700
Committer: Miklos Szegedi <szege...@apache.org>
Committed: Thu May 17 11:16:04 2018 -0700

----------------------------------------------------------------------
 .../mapred/LocalDistributedCacheManager.java    | 31 ++++++++++++++++----
 .../apache/hadoop/mapred/LocalJobRunner.java    | 14 ++++++---
 2 files changed, 36 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2cdffb9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
index bcf73d1..1565e2e 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
@@ -73,6 +73,7 @@ class LocalDistributedCacheManager {
   private List<String> localClasspaths = new ArrayList<String>();
   
   private List<File> symlinksCreated = new ArrayList<File>();
+  private URLClassLoader classLoaderCreated = null;
   
   private boolean setupCalled = false;
   
@@ -82,7 +83,7 @@ class LocalDistributedCacheManager {
    * @param conf
    * @throws IOException
    */
-  public void setup(JobConf conf, JobID jobId) throws IOException {
+  public synchronized void setup(JobConf conf, JobID jobId) throws IOException 
{
     File workDir = new File(System.getProperty("user.dir"));
     
     // Generate YARN local resources objects corresponding to the distributed
@@ -212,7 +213,7 @@ class LocalDistributedCacheManager {
    * Should be called after setup().
    * 
    */
-  public boolean hasLocalClasspaths() {
+  public synchronized boolean hasLocalClasspaths() {
     if (!setupCalled) {
       throw new IllegalStateException(
           "hasLocalClasspaths() should be called after setup()");
@@ -224,8 +225,11 @@ class LocalDistributedCacheManager {
    * Creates a class loader that includes the designated
    * files and archives.
    */
-  public ClassLoader makeClassLoader(final ClassLoader parent)
+  public synchronized ClassLoader makeClassLoader(final ClassLoader parent)
       throws MalformedURLException {
+    if (classLoaderCreated != null) {
+      throw new IllegalStateException("A classloader was already created");
+    }
     final URL[] urls = new URL[localClasspaths.size()];
     for (int i = 0; i < localClasspaths.size(); ++i) {
       urls[i] = new File(localClasspaths.get(i)).toURI().toURL();
@@ -234,12 +238,29 @@ class LocalDistributedCacheManager {
     return AccessController.doPrivileged(new PrivilegedAction<ClassLoader>() {
       @Override
       public ClassLoader run() {
-        return new URLClassLoader(urls, parent);
+        classLoaderCreated = new URLClassLoader(urls, parent);
+        return classLoaderCreated;
       }
     });
   }
 
-  public void close() throws IOException {
+  public synchronized void close() throws IOException {
+    if(classLoaderCreated != null) {
+      AccessController.doPrivileged(new PrivilegedAction<Void>() {
+        @Override
+        public Void run() {
+          try {
+            classLoaderCreated.close();
+            classLoaderCreated = null;
+          } catch (IOException e) {
+            LOG.warn("Failed to close classloader created " +
+                "by LocalDistributedCacheManager");
+          }
+          return null;
+        }
+      });
+    }
+
     for (File symlink : symlinksCreated) {
       if (!symlink.delete()) {
         LOG.warn("Failed to delete symlink created by the local job runner: " +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2cdffb9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 2ab4e76..0f1d759 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -593,10 +593,16 @@ public class LocalJobRunner implements ClientProtocol {
 
       } finally {
         try {
-          fs.delete(systemJobFile.getParent(), true);  // delete submit dir
-          localFs.delete(localJobFile, true);              // delete local copy
-          // Cleanup distributed cache
-          localDistributedCacheManager.close();
+          try {
+            // Cleanup distributed cache
+            localDistributedCacheManager.close();
+          } finally {
+            try {
+              fs.delete(systemJobFile.getParent(), true); // delete submit dir
+            } finally {
+              localFs.delete(localJobFile, true);         // delete local copy
+            }
+          }
         } catch (IOException e) {
           LOG.warn("Error cleaning up "+id+": "+e);
         }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

Reply via email to