IGNITE-3929: Implementation.

Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/423141dc
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/423141dc
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/423141dc

Branch: refs/heads/ignite-3929-1
Commit: 423141dca3ffb200f0a93f0e136f3d15c1ee1a0f
Parents: 64e39ec
Author: vozerov-gridgain <voze...@gridgain.com>
Authored: Tue Sep 20 16:34:52 2016 +0300
Committer: vozerov-gridgain <voze...@gridgain.com>
Committed: Tue Sep 20 16:34:52 2016 +0300

----------------------------------------------------------------------
 .../processors/hadoop/HadoopClassLoader.java    | 190 +++++++++++++++----
 1 file changed, 148 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/423141dc/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 5297cea..d13bf97 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.hadoop;
 
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.util.ClassCache;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -25,12 +26,15 @@ import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.Map;
 import java.util.UUID;
 import java.util.Vector;
@@ -43,20 +47,12 @@ import java.util.concurrent.ConcurrentMap;
  * unavailable for parent.
  */
 public class HadoopClassLoader extends URLClassLoader implements ClassCache {
-    static {
-        // We are very parallel capable.
-        registerAsParallelCapable();
-    }
-
     /** Hadoop class name: Daemon. */
     public static final String CLS_DAEMON = "org.apache.hadoop.util.Daemon";
 
     /** Hadoop class name: ShutdownHookManager. */
     public static final String CLS_SHUTDOWN_HOOK_MANAGER = 
"org.apache.hadoop.util.ShutdownHookManager";
 
-    /** Hadoop class name: NativeCodeLoader. */
-    public static final String CLS_NATIVE_CODE_LOADER = 
"org.apache.hadoop.util.NativeCodeLoader";
-
     /** Hadoop class name: Daemon replacement. */
     public static final String CLS_DAEMON_REPLACE = 
"org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon";
 
@@ -64,15 +60,21 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
     public static final String CLS_SHUTDOWN_HOOK_MANAGER_REPLACE =
         
"org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager";
 
-    /** Name of libhadoop library. */
-    private static final String LIBHADOOP = "hadoop.";
-
     /** */
     private static final URLClassLoader APP_CLS_LDR = 
(URLClassLoader)HadoopClassLoader.class.getClassLoader();
 
     /** */
     private static final Collection<URL> appJars = 
F.asList(APP_CLS_LDR.getURLs());
 
+    /** Mutex for native libraries initialization. */
+    private static final Object LIBS_MUX = new Object();
+
+    /** Predefined native libraries to load. */
+    private static final Collection<String> PREDEFINED_NATIVE_LIBS;
+
+    /** Native libraries. */
+    private static Collection<Object> NATIVE_LIBS;
+
     /** */
     private static volatile Collection<URL> hadoopJars;
 
@@ -86,12 +88,18 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
     @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
     private final String name;
 
-    /** Native library names. */
-    private final String[] libNames;
-
     /** Igfs Helper. */
     private final HadoopHelper helper;
 
+    static {
+        // We are very parallel capable.
+        registerAsParallelCapable();
+
+        PREDEFINED_NATIVE_LIBS = new HashSet<>();
+
+        PREDEFINED_NATIVE_LIBS.add("hadoop");
+    }
+
     /**
      * Gets name for Job class loader. The name is specific for local node id.
      * @param locNodeId The local node id.
@@ -127,10 +135,9 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
         assert !(getParent() instanceof HadoopClassLoader);
 
         this.name = name;
-        this.libNames = libNames;
         this.helper = helper;
 
-        initializeNativeLibraries();
+        initializeNativeLibraries(libNames);
     }
 
     /**
@@ -145,49 +152,126 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
      * @see <a 
href="http://docs.oracle.com/javase/1.5.0/docs/guide/jni/spec/invocation.html#library_version";>
      *     JNI specification</a>
      */
-    private void initializeNativeLibraries() {
-        try {
-            // This must trigger native library load.
-            // TODO: Do not delegate to APP LDR
-            Class.forName(CLS_NATIVE_CODE_LOADER, true, APP_CLS_LDR);
+    private void initializeNativeLibraries(@Nullable String[] usrLibs) {
+        Collection<Object> res;
+
+        synchronized (LIBS_MUX) {
+            if (NATIVE_LIBS == null) {
+                LinkedList<NativeLibrary> libs = new LinkedList<>();
+
+                for (String lib : PREDEFINED_NATIVE_LIBS)
+                    libs.add(new NativeLibrary(lib, true));
+
+                if (!F.isEmpty(usrLibs)) {
+                    for (String usrLib : usrLibs)
+                        libs.add(new NativeLibrary(usrLib, false));
+                }
+
+                NATIVE_LIBS = initializeNativeLibraries0(libs);
+            }
 
-            final Vector<Object> curVector = U.field(this, "nativeLibraries");
+            res = NATIVE_LIBS;
+        }
+
+        // Link libraries to class loader.
+        Vector<Object> ldrLibs = nativeLibraries(this);
+
+        ldrLibs.addAll(res);
+    }
 
-            // TODO: Do not delegate to APP LDR
-            ClassLoader ldr = APP_CLS_LDR;
+    /**
+     * Initialize native libraries.
+     *
+     * @param libs Libraries to initialize.
+     * @return Initialized libraries.
+     */
+    private static Collection<Object> 
initializeNativeLibraries0(Collection<NativeLibrary> libs) {
+        assert Thread.holdsLock(LIBS_MUX);
 
-            while (ldr != null) {
-                Vector vector = U.field(ldr, "nativeLibraries");
+        Collection<Object> res = new HashSet<>();
 
-                for (Object lib : vector) {
-                    String name = U.field(lib, "name");
+        for (NativeLibrary lib : libs) {
+            String libName = lib.name;
 
-                    boolean add = name.contains(LIBHADOOP);
+            File libFile = new File(libName);
 
-                    if (!add && libNames != null) {
-                        for (String libName : libNames) {
-                            if (libName != null && name.contains(libName)) {
-                                add = true;
+            try {
+                // Load library.
+                if (libFile.isAbsolute())
+                    System.load(libName);
+                else
+                    System.loadLibrary(libName);
+
+                // Find library in class loader internals.
+                Object libObj = null;
+
+                ClassLoader ldr = HadoopClassLoader.class.getClassLoader();
+
+                while (ldr != null) {
+                    for (Object ldrLibObj : nativeLibraries(ldr)) {
+                        String name = nativeLibraryName(ldrLibObj);
+
+                        if (libFile.isAbsolute()) {
+                            if (F.eq(name, libFile.getCanonicalPath())) {
+                                libObj = ldrLibObj;
+
+                                break;
+                            }
+                        }
+                        else {
+                            if (name.contains(libName)) {
+                                libObj = ldrLibObj;
 
                                 break;
                             }
                         }
                     }
 
-                    if (add) {
-                        curVector.add(lib);
+                    if (libObj != null)
+                        break;
 
-                        return;
-                    }
+                    ldr = ldr.getParent();
                 }
 
-                ldr = ldr.getParent();
+                if (libObj == null)
+                    throw new IgniteException("Failed to find loaded library: 
" + libName);
+
+                res.add(libObj);
+            }
+            catch (UnsatisfiedLinkError e) {
+                if (!lib.optional)
+                    throw e;
+            }
+            catch (IOException e) {
+                throw new IgniteException("Failed to initialize native 
librarires due to unexpected exception.", e);
             }
         }
-        catch (Exception e) {
-            U.quietAndWarn(null, "Failed to initialize Hadoop native library " 
+
-                "(native Hadoop methods might not work properly): " + e);
-        }
+
+        return res;
+    }
+
+    /**
+     * Get native libraries collection for the given class loader.
+     *
+     * @param ldr Class loaded.
+     * @return Native libraries.
+     */
+    private static Vector<Object> nativeLibraries(ClassLoader ldr) {
+        assert ldr != null;
+
+        return U.field(ldr, "nativeLibraries");
+    }
+
+    /**
+     * Get native library name.
+     *
+     * @param lib Library.
+     * @return Name.
+     */
+    private static String nativeLibraryName(Object lib) {
+        assert lib != null;
+
+        return U.field(lib, "name");
     }
 
     /** {@inheritDoc} */
@@ -364,4 +448,26 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
     public String name() {
         return name;
     }
+
+    /**
+     * Native library abstraction.
+     */
+    private static class NativeLibrary {
+        /** Library name. */
+        private final String name;
+
+        /** Whether library is optional. */
+        private final boolean optional;
+
+        /**
+         * Constructor.
+         *
+         * @param name Library name.
+         * @param optional Optional flag.
+         */
+        public NativeLibrary(String name, boolean optional) {
+            this.name = name;
+            this.optional = optional;
+        }
+    }
 }
\ No newline at end of file

Reply via email to