Repository: ignite
Updated Branches:
  refs/heads/ignite-1.6.8-hadoop 60c96ef3c -> 0f7ccb3d0


IGNITE-3941: Introduced common HadoopClassLoader. This closes #1091.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0f7ccb3d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0f7ccb3d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0f7ccb3d

Branch: refs/heads/ignite-1.6.8-hadoop
Commit: 0f7ccb3d064cfa9be3cbd18bfa8bc74c32af52f0
Parents: 60c96ef
Author: vozerov-gridgain <voze...@gridgain.com>
Authored: Tue Sep 20 18:06:39 2016 +0300
Committer: vozerov-gridgain <voze...@gridgain.com>
Committed: Tue Sep 20 18:06:39 2016 +0300

----------------------------------------------------------------------
 .../apache/ignite/internal/IgniteKernal.java    |  2 +-
 .../processors/hadoop/HadoopClassLoader.java    | 10 ---
 .../processors/hadoop/HadoopHelper.java         |  7 ++
 .../processors/hadoop/HadoopNoopHelper.java     | 17 +++++
 .../hadoop/jobtracker/HadoopJobTracker.java     | 10 +--
 .../processors/hadoop/HadoopHelperImpl.java     | 78 +++++++++++++-------
 6 files changed, 76 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java 
b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 4f916be..c909de5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -822,7 +822,7 @@ public class IgniteKernal implements IgniteEx, 
IgniteMXBean, Externalizable {
 
             
addHelper(IGFS_HELPER.create(F.isEmpty(cfg.getFileSystemConfiguration())));
 
-            addHelper(HADOOP_HELPER.createIfInClassPath(null, false));
+            addHelper(HADOOP_HELPER.createIfInClassPath(ctx, false));
 
             startProcessor(new IgnitePluginProcessor(ctx, cfg, plugins));
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 3bf841b..90a2529 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -36,7 +36,6 @@ import java.util.Collection;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Map;
-import java.util.UUID;
 import java.util.Vector;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -101,15 +100,6 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
     }
 
     /**
-     * Gets name for Job class loader. The name is specific for local node id.
-     * @param locNodeId The local node id.
-     * @return The class loader name.
-     */
-    public static String nameForJob(UUID locNodeId) {
-        return "hadoop-job-node-" + locNodeId.toString();
-    }
-
-    /**
      * Gets name for the task class loader. Task class loader
      * @param info The task info.
      * @param prefix Get only prefix (without task type and number)

http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
index 40852c5..9297a2f 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
@@ -25,6 +25,13 @@ import org.jetbrains.annotations.Nullable;
  */
 public interface HadoopHelper {
     /**
+     * Get common Hadoop class loader.
+     *
+     * @return Common Hadoop class loader.
+     */
+    public HadoopClassLoader commonClassLoader();
+
+    /**
      * Load special replacement and impersonate.
      *
      * @param in Input stream.

http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
index d9ce857..32abc91 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.hadoop;
 
+import org.apache.ignite.internal.GridKernalContext;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.InputStream;
@@ -24,7 +25,23 @@ import java.io.InputStream;
 /**
  * Noop Hadoop Helper implementation.
  */
+@SuppressWarnings("unused")
 public class HadoopNoopHelper implements HadoopHelper {
+    /**
+     * Constructor.
+     *
+     * @param ctx Kernal context.
+     */
+    @SuppressWarnings("UnusedParameters")
+    public HadoopNoopHelper(GridKernalContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public HadoopClassLoader commonClassLoader() {
+        throw unsupported();
+    }
+
     /** {@inheritDoc} */
     @Override public boolean hasExternalDependencies(String clsName, 
ClassLoader parentClsLdr) {
         throw unsupported();

http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
 
b/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
index a2c55a2..cb69823 100644
--- 
a/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
+++ 
b/modules/hadoop-impl/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
@@ -153,17 +153,9 @@ public class HadoopJobTracker extends HadoopComponent {
 
         evtProcSvc = Executors.newFixedThreadPool(1);
 
-        UUID nodeId = ctx.localNodeId();
-
         assert jobCls == null;
 
-        String[] libNames = null;
-
-        if (ctx.configuration() != null)
-            libNames = ctx.configuration().getNativeLibraryNames();
-
-        HadoopClassLoader ldr = new HadoopClassLoader(null, 
HadoopClassLoader.nameForJob(nodeId), libNames,
-            ctx.kernalContext().hadoopHelper());
+        HadoopClassLoader ldr = 
ctx.kernalContext().hadoopHelper().commonClassLoader();
 
         try {
             jobCls = 
(Class<HadoopV2Job>)ldr.loadClass(HadoopV2Job.class.getName());

http://git-wip-us.apache.org/repos/asf/ignite/blob/0f7ccb3d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
index 59a37e9..0ebe645 100644
--- 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
+++ 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelperImpl.java
@@ -16,6 +16,7 @@
  */
 package org.apache.ignite.internal.processors.hadoop;
 
+import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.util.typedef.F;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -46,14 +47,53 @@ public class HadoopHelperImpl implements HadoopHelper {
     /** Cache for resolved dependency info. */
     private static final Map<String, Boolean> dependenciesCache = new 
ConcurrentHashMap8<>();
 
+    /** Kernal context. */
+    private final GridKernalContext ctx;
+
+    /** Common class loader. */
+    private volatile HadoopClassLoader ldr;
+
     /**
-     * Load special replacement and impersonate
+     * Default constructor.
+     */
+    public HadoopHelperImpl() {
+        this(null);
+    }
+
+    /**
+     * Constructor.
      *
-     * @param in Input stream.
-     * @param originalName Original class name.
-     * @param replaceName Replacer class name.
-     * @return Result.
+     * @param ctx Kernal context.
      */
+    public HadoopHelperImpl(GridKernalContext ctx) {
+        this.ctx = ctx;
+    }
+
+    /** {@inheritDoc} */
+    @Override public HadoopClassLoader commonClassLoader() {
+        HadoopClassLoader res = ldr;
+
+        if (res == null) {
+            synchronized (this) {
+                res = ldr;
+
+                if (res == null) {
+                    String[] libNames = null;
+
+                    if (ctx != null && ctx.config().getHadoopConfiguration() 
!= null)
+                        libNames = 
ctx.config().getHadoopConfiguration().getNativeLibraryNames();
+
+                    res = new HadoopClassLoader(null, "hadoop-common", 
libNames, this);
+
+                    ldr = res;
+                }
+            }
+        }
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
     @Override public byte[] loadReplace(InputStream in, final String 
originalName, final String replaceName) {
         ClassReader rdr;
 
@@ -84,20 +124,12 @@ public class HadoopHelperImpl implements HadoopHelper {
         return w.toByteArray();
     }
 
-    /**
-     * @param cls Class name.
-     * @return {@code true} If this is Hadoop class.
-     */
+    /** {@inheritDoc} */
     @Override public boolean isHadoop(String cls) {
         return cls.startsWith("org.apache.hadoop.");
     }
 
-    /**
-     * Need to parse only Ignite Hadoop and IGFS classes.
-     *
-     * @param cls Class name.
-     * @return {@code true} if we need to check this class.
-     */
+    /** {@inheritDoc} */
     @Override public boolean isHadoopIgfs(String cls) {
         String ignitePkgPrefix = "org.apache.ignite";
 
@@ -109,22 +141,12 @@ public class HadoopHelperImpl implements HadoopHelper {
                 cls.indexOf("hadoop.", len) != -1);
     }
 
-    /**
-     * @param ldr Loader.
-     * @param clsName Class.
-     * @return Input stream.
-     */
+    /** {@inheritDoc} */
     @Override @Nullable public InputStream loadClassBytes(ClassLoader ldr, 
String clsName) {
         return ldr.getResourceAsStream(clsName.replace('.', '/') + ".class");
     }
 
-    /**
-     * Check whether class has external dependencies on Hadoop.
-     *
-     * @param clsName Class name.
-     * @param parentClsLdr Parent class loader.
-     * @return {@code True} if class has external dependencies.
-     */
+    /** {@inheritDoc} */
     @Override public boolean hasExternalDependencies(String clsName, 
ClassLoader parentClsLdr) {
         Boolean hasDeps = dependenciesCache.get(clsName);
 
@@ -152,7 +174,7 @@ public class HadoopHelperImpl implements HadoopHelper {
      * @param ctx Context.
      * @return {@code true} If the class has external dependencies.
      */
-    boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr, 
CollectingContext ctx) {
+    private boolean hasExternalDependencies(String clsName, ClassLoader 
parentClsLdr, CollectingContext ctx) {
         if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea 
sucks, so filtering explicitly as external.
             return true;
 

Reply via email to