3929: wip, current state.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/83ff85f3 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/83ff85f3 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/83ff85f3 Branch: refs/heads/ignite-3929-1 Commit: 83ff85f3babd3bedb70fda86703146d2ada393a0 Parents: df3ad76 Author: iveselovskiy <iveselovs...@gridgain.com> Authored: Tue Sep 20 16:03:35 2016 +0300 Committer: iveselovskiy <iveselovs...@gridgain.com> Committed: Tue Sep 20 16:03:35 2016 +0300 ---------------------------------------------------------------------- .../processors/hadoop/HadoopSnappyTest.java | 2 +- .../processors/hadoop/HadoopClassLoader.java | 51 ++++++++++++++------ .../internal/processors/hadoop/LoadHelper.java | 31 +++++++----- 3 files changed, 56 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/83ff85f3/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java index 0b86c50..ca47334 100644 --- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java +++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java @@ -58,7 +58,7 @@ public class HadoopSnappyTest extends GridCommonAbstractTest { U.invoke(cls, null, "checkSnappy"); } - //checkSnappy(); + checkSnappy(); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/83ff85f3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java index c6d2296..276b22d 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.hadoop; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; @@ -114,6 +115,8 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { return "hadoop-task-" + info.jobId() + "-" + info.type() + "-" + info.taskNumber(); } + private static final String[] DEFAULT_OPTIONAL_LIBS_TO_BE_LOADED = { "hadoop" }; + /** * Constructor. * @@ -202,20 +205,30 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { * be loaded only once. */ private static void setNativeLibrariesToBeInjectedIfNeeded(HadoopClassLoader instance, String[] libs) { + System.out.println("### Loading libs: " + libs); + if (libs == null) return; - - boolean created = false; +// +// boolean created = false; // 1. If needed, init the native lib data collection: if (nativeLibrariesToBeInjected == null) { synchronized (HadoopClassLoader.class) { if (nativeLibrariesToBeInjected == null) { + LinkedHashSet<Object> natives0 = getNativeLibraries(APP_CLS_LDR); + instance.runLoadingCode(libs); - nativeLibrariesToBeInjected = instance.collectNativeLibraries(); + LinkedHashSet<Object> natives1 = getNativeLibraries(APP_CLS_LDR); - created = true; + natives1.removeAll(natives0); + + nativeLibrariesToBeInjected = Collections.unmodifiableCollection(natives1); + + System.out.println("### Collected loaded libraries: " + nativeLibrariesToBeInjected); +// +// created = true; } } } @@ -223,9 +236,19 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { assert nativeLibrariesToBeInjected != null; // 2. Inject libraries: - if (!created) + //if (!created) // This is an instance that did not load the libs: - instance.injectNatives(); + instance.injectNatives(); + } + + /** + * + * @return + */ + private static LinkedHashSet<Object> getNativeLibraries(ClassLoader cl) { + Vector<Object> curVector = U.field(cl, "nativeLibraries"); + + return new LinkedHashSet<>(curVector); } /** @@ -251,7 +274,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { private Collection<Object> collectNativeLibraries() { List<Object> target = new ArrayList<>(); - ClassLoader ldr = this; + ClassLoader ldr = APP_CLS_LDR; while (ldr != null) { collectNativeLibrariesFromLoader(ldr, target); @@ -267,17 +290,17 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { */ private void runLoadingCode(String[] libs) { try { - // TODO: "XXX" is a special class loaded by Hadoop class loader (simulating Hadoop class). - // NB: this sample class must *not* cause loading of any natives. - Class<?> sampleCls = this.loadClass(XXX.class.getName(), true); - - assert sampleCls != null; - assert sampleCls.getClassLoader() == this; +// // TODO: "XXX" is a special class loaded by Hadoop class loader (simulating Hadoop class). +// // NB: this sample class must *not* cause loading of any natives. +// Class<?> sampleCls = this.loadClass(XXX.class.getName(), true); +// +// assert sampleCls != null; +// assert sampleCls.getClassLoader() == this; Collection<String> loadedLibs = new ArrayList<>(); for (String lib: libs) { - boolean ok = LoadHelper.tryLoad(sampleCls, lib); + boolean ok = LoadHelper.tryLoad(null, lib); if (ok) loadedLibs.add(lib); http://git-wip-us.apache.org/repos/asf/ignite/blob/83ff85f3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java index a795085..223855b 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java @@ -7,18 +7,18 @@ import java.lang.reflect.Method; */ public class LoadHelper { - private static Method method; - - static { - try { - method = ClassLoader.class.getDeclaredMethod("loadLibrary", - new Class[] {Class.class, String.class, boolean.class}); - - method.setAccessible(true); - } catch (Exception e) { - e.printStackTrace(); - } - } +// private static Method method; +// +// static { +// try { +// method = ClassLoader.class.getDeclaredMethod("loadLibrary", +// new Class[] {Class.class, String.class, boolean.class}); +// +// method.setAccessible(true); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } /** * Utility method that loads given class by name with the given "caller" class. @@ -27,7 +27,7 @@ public class LoadHelper { */ public static boolean tryLoad(Class caller, String libName) { try { - method.invoke(null, new Object[] {caller/*caller class*/, libName /*lib*/ , false/*isAbsolute*/ }); + System.loadLibrary(libName); return true; } @@ -39,4 +39,9 @@ public class LoadHelper { } } + + public static void main(String[] args) { + System.load("/home/ivan/hadoop-2.6.0/lib/native/libhadoop.so"); + } + }
