Merge branch 'master' into ignite-2218 # Conflicts: # modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java # modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/86287828 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/86287828 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/86287828 Branch: refs/heads/ignite-2218 Commit: 8628782873b7a46b8521f825bf6a20b6e3358fbb Parents: 7f8ca71 012ca73 Author: vozerov-gridgain <[email protected]> Authored: Mon Jan 4 10:55:12 2016 +0400 Committer: vozerov-gridgain <[email protected]> Committed: Mon Jan 4 10:55:12 2016 +0400 ---------------------------------------------------------------------- .../processors/hadoop/HadoopClassLoader.java | 631 ++++++++++++++----- .../hadoop/HadoopClassLoaderTest.java | 101 ++- .../hadoop/deps/CircularWIthHadoop.java | 32 + .../hadoop/deps/CircularWithoutHadoop.java | 27 + .../processors/hadoop/deps/WithCast.java | 41 ++ .../hadoop/deps/WithClassAnnotation.java | 28 + .../hadoop/deps/WithConstructorInvocation.java | 31 + .../processors/hadoop/deps/WithExtends.java | 27 + .../processors/hadoop/deps/WithField.java | 29 + .../processors/hadoop/deps/WithImplements.java | 36 ++ .../hadoop/deps/WithIndirectField.java | 27 + .../processors/hadoop/deps/WithInitializer.java | 33 + .../processors/hadoop/deps/WithInnerClass.java | 31 + .../hadoop/deps/WithLocalVariable.java | 38 ++ .../hadoop/deps/WithMethodAnnotation.java | 32 + .../hadoop/deps/WithMethodArgument.java | 31 + .../hadoop/deps/WithMethodCheckedException.java | 31 + .../hadoop/deps/WithMethodInvocation.java | 31 + .../hadoop/deps/WithMethodReturnType.java | 31 + .../hadoop/deps/WithMethodRuntimeException.java | 31 + .../processors/hadoop/deps/WithOuterClass.java | 38 ++ .../hadoop/deps/WithParameterAnnotation.java | 31 + .../processors/hadoop/deps/WithStaticField.java | 29 + .../hadoop/deps/WithStaticInitializer.java | 34 + .../processors/hadoop/deps/Without.java | 25 + .../testsuites/IgniteHadoopTestSuite.java | 3 + 26 files changed, 1275 insertions(+), 184 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/86287828/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java ---------------------------------------------------------------------- diff --cc modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java index f605c20,735133f..270b31d --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java @@@ -30,11 -30,10 +30,10 @@@ import java.util.HashSet import java.util.Map; import java.util.Set; import java.util.UUID; - +import java.util.Vector; - import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.util.NativeCodeLoader; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon; -import org.apache.ignite.internal.processors.hadoop.v2.HadoopNativeCodeLoader; import org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; @@@ -316,10 -273,30 +320,30 @@@ public class HadoopClassLoader extends } /** + * Check whether class has external dependencies on Hadoop. - * ++ * + * @param clsName Class name. + * @return {@code True} if class has external dependencies. + */ + boolean hasExternalDependencies(String clsName) { + CollectingContext ctx = new CollectingContext(); + + ctx.annVisitor = new CollectingAnnotationVisitor(ctx); + ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor); + ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor); + ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor); - ++ + return hasExternalDependencies(clsName, ctx); + } - ++ + /** + * Check whether class has external dependencies on Hadoop. - * ++ * * @param clsName Class name. - * @param ctx Context. ++ * @param ctx Context. * @return {@code true} If the class has external dependencies. */ - boolean hasExternalDependencies(final String clsName, final Set<String> visited) { + boolean hasExternalDependencies(String clsName, CollectingContext ctx) { if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external. return true; @@@ -666,4 -500,446 +547,446 @@@ public String name() { return name; } + + /** + * Context for dependencies collection. + */ + private class CollectingContext { + /** Visited classes. */ + private final Set<String> visited = new HashSet<>(); + + /** Whether dependency found. */ + private boolean found; + + /** Annotation visitor. */ + private AnnotationVisitor annVisitor; + + /** Method visitor. */ + private MethodVisitor mthdVisitor; + + /** Field visitor. */ + private FieldVisitor fldVisitor; - ++ + /** Class visitor. */ + private ClassVisitor clsVisitor; + + /** + * Processes a method descriptor + * @param methDesc The method desc String. + */ + void onMethodsDesc(final String methDesc) { + // Process method return type: + onType(Type.getReturnType(methDesc)); + + if (found) + return; + + // Process method argument types: + for (Type t: Type.getArgumentTypes(methDesc)) { + onType(t); + + if (found) + return; + } + } + + /** + * Processes dependencies of a class. + * + * @param depCls The class name as dot-notated FQN. + */ + void onClass(final String depCls) { + assert depCls.indexOf('/') == -1 : depCls; // class name should be fully converted to dot notation. + assert depCls.charAt(0) != 'L' : depCls; + assert validateClassName(depCls) : depCls; + + if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // Filter out platform classes. + return; + + if (visited.contains(depCls)) + return; + + Boolean res = cache.get(depCls); + + if (res == Boolean.TRUE || (res == null && hasExternalDependencies(depCls, this))) + found = true; + } + + /** + * Analyses dependencies of given type. + * + * @param t The type to process. + */ + void onType(Type t) { + if (t == null) + return; + + int sort = t.getSort(); + + switch (sort) { + case Type.ARRAY: + onType(t.getElementType()); + + break; + + case Type.OBJECT: + onClass(t.getClassName()); + + break; + } + } + + /** + * Analyses dependencies of given object type. + * + * @param objType The object type to process. + */ + void onInternalTypeName(String objType) { + if (objType == null) + return; + + assert objType.length() > 1 : objType; + + if (objType.charAt(0) == '[') + // handle array. In this case this is a type descriptor notation, like "[Ljava/lang/Object;" + onType(objType); + else { + assert objType.indexOf('.') == -1 : objType; // Must be slash-separated FQN. + + String clsName = objType.replace('/', '.'); // Convert it to dot notation. + + onClass(clsName); // Process. + } + } + + /** + * Type description analyser. + * + * @param desc The description. + */ + void onType(String desc) { + if (!F.isEmpty(desc)) { + if (desc.length() <= 1) + return; // Optimization: filter out primitive types in early stage. + + Type t = Type.getType(desc); + + onType(t); + } + } - } ++ } + + /** + * Annotation visitor. + */ + private static class CollectingAnnotationVisitor extends AnnotationVisitor { + /** */ + final CollectingContext ctx; + + /** + * Annotation visitor. - * ++ * + * @param ctx The collector. + */ + CollectingAnnotationVisitor(CollectingContext ctx) { + super(Opcodes.ASM4); + + this.ctx = ctx; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotation(String name, String desc) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return this; + } + + /** {@inheritDoc} */ + @Override public void visitEnum(String name, String desc, String val) { + if (ctx.found) + return; + + ctx.onType(desc); + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitArray(String name) { + return ctx.found ? null : this; + } + + /** {@inheritDoc} */ + @Override public void visit(String name, Object val) { + if (ctx.found) + return; + + if (val instanceof Type) + ctx.onType((Type)val); + } + + /** {@inheritDoc} */ + @Override public void visitEnd() { + // No-op. + } + } + + /** + * Field visitor. + */ + private static class CollectingFieldVisitor extends FieldVisitor { + /** Collector. */ + private final CollectingContext ctx; + + /** Annotation visitor. */ + private final AnnotationVisitor av; + + /** + * Constructor. + */ + CollectingFieldVisitor(CollectingContext ctx, AnnotationVisitor av) { + super(Opcodes.ASM4); + + this.ctx = ctx; + this.av = av; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public void visitAttribute(Attribute attr) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void visitEnd() { + // No-op. + } + } + + /** + * Class visitor. + */ + private static class CollectingClassVisitor extends ClassVisitor { + /** Collector. */ + private final CollectingContext ctx; + + /** Annotation visitor. */ + private final AnnotationVisitor av; + + /** Method visitor. */ + private final MethodVisitor mv; + + /** Field visitor. */ + private final FieldVisitor fv; + + /** + * Constructor. + * + * @param ctx Collector. + * @param av Annotation visitor. + * @param mv Method visitor. + * @param fv Field visitor. + */ + CollectingClassVisitor(CollectingContext ctx, AnnotationVisitor av, MethodVisitor mv, FieldVisitor fv) { + super(Opcodes.ASM4); + + this.ctx = ctx; + this.av = av; + this.mv = mv; + this.fv = fv; + } + + /** {@inheritDoc} */ + @Override public void visit(int i, int i2, String name, String signature, String superName, String[] ifaces) { + if (ctx.found) + return; + + ctx.onInternalTypeName(superName); + + if (ctx.found) + return; + + if (ifaces != null) { + for (String iface : ifaces) { + ctx.onInternalTypeName(iface); + + if (ctx.found) + return; + } + } + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public void visitInnerClass(String name, String outerName, String innerName, int i) { + if (ctx.found) + return; + + ctx.onInternalTypeName(name); + } + + /** {@inheritDoc} */ + @Override public FieldVisitor visitField(int i, String name, String desc, String signature, Object val) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : fv; + } + + /** {@inheritDoc} */ + @Override public MethodVisitor visitMethod(int i, String name, String desc, String signature, + String[] exceptions) { + if (ctx.found) + return null; + + ctx.onMethodsDesc(desc); + + // Process declared method exceptions: + if (exceptions != null) { + for (String e : exceptions) + ctx.onInternalTypeName(e); + } + + return ctx.found ? null : mv; + } + } + + /** + * Method visitor. + */ + private static class CollectingMethodVisitor extends MethodVisitor { + /** Collector. */ + private final CollectingContext ctx; + + /** Annotation visitor. */ + private final AnnotationVisitor av; + + /** + * Constructor. + * + * @param ctx Collector. + * @param av Annotation visitor. + */ + private CollectingMethodVisitor(CollectingContext ctx, AnnotationVisitor av) { + super(Opcodes.ASM4); + + this.ctx = ctx; + this.av = av; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitParameterAnnotation(int i, String desc, boolean b) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotationDefault() { + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public void visitFieldInsn(int opcode, String owner, String name, String desc) { + if (ctx.found) + return; + + ctx.onInternalTypeName(owner); + + if (ctx.found) + return; + + ctx.onType(desc); + } + + /** {@inheritDoc} */ + @Override public void visitInvokeDynamicInsn(String name, String desc, Handle bsm, Object... bsmArgs) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void visitFrame(int type, int nLoc, Object[] locTypes, int nStack, Object[] stackTypes) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void visitLocalVariable(String name, String desc, String signature, Label lb, + Label lb2, int i) { + if (ctx.found) + return; + + ctx.onType(desc); + } + + /** {@inheritDoc} */ + @Override public void visitMethodInsn(int i, String owner, String name, String desc) { + if (ctx.found) + return; + + ctx.onInternalTypeName(owner); + + if (ctx.found) + return; + + ctx.onMethodsDesc(desc); + } + + /** {@inheritDoc} */ + @Override public void visitMultiANewArrayInsn(String desc, int dim) { + if (ctx.found) + return; + + ctx.onType(desc); + } + + /** {@inheritDoc} */ + @Override public void visitTryCatchBlock(Label start, Label end, Label hndl, String typeStr) { + if (ctx.found) + return; + + ctx.onInternalTypeName(typeStr); + } + + /** {@inheritDoc} */ + @Override public void visitTypeInsn(int opcode, String type) { + if (ctx.found) + return; + + ctx.onInternalTypeName(type); + } + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/86287828/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java ---------------------------------------------------------------------- diff --cc modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java index 034e934,1831085..6c542b5 --- a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java @@@ -99,9 -96,8 +100,11 @@@ public class IgniteHadoopTestSuite exte TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite"); + suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyTest.class.getName()))); + suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyFullMapReduceTest.class.getName()))); + + suite.addTest(new TestSuite(ldr.loadClass(HadoopClassLoaderTest.class.getName()))); + suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfs20FileSystemLoopbackPrimarySelfTest.class.getName()))); suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfsDualSyncSelfTest.class.getName())));
