IGNITE-3912: Preliminary changes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2fe0272b Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2fe0272b Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2fe0272b Branch: refs/heads/ignite-1.6.8-hadoop Commit: 2fe0272ba5f2c9003ab57844ba730a384bb67ca1 Parents: 2474e2b Author: vozerov-gridgain <voze...@gridgain.com> Authored: Fri Sep 16 12:55:00 2016 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Fri Sep 16 12:55:00 2016 +0300 ---------------------------------------------------------------------- bin/include/setenv.bat | 2 +- bin/include/setenv.sh | 2 +- .../processors/hadoop/HadoopClasspathMain.java | 44 ++ .../processors/hadoop/HadoopClasspathUtils.java | 461 ++++++++++++ .../processors/hadoop/HadoopLocations.java | 123 ++++ .../processors/hadoop/HadoopClassLoader.java | 697 ++----------------- .../hadoop/HadoopClassLoaderUtils.java | 684 ++++++++++++++++++ .../processors/hadoop/HadoopClasspathMain.java | 44 -- .../processors/hadoop/HadoopClasspathUtils.java | 461 ------------ .../processors/hadoop/HadoopLocations.java | 123 ---- .../processors/hadoop/v2/HadoopV2Job.java | 2 +- .../hadoop/HadoopClassLoaderTest.java | 4 +- 12 files changed, 1365 insertions(+), 1282 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/bin/include/setenv.bat ---------------------------------------------------------------------- diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat index 9d55521..2e8eac6 100644 --- a/bin/include/setenv.bat +++ b/bin/include/setenv.bat @@ -47,7 +47,7 @@ if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1 if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS% -if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\ignite-hadoop\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i +if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/bin/include/setenv.sh ---------------------------------------------------------------------- diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh index a85cba3..54f5831 100755 --- a/bin/include/setenv.sh +++ b/bin/include/setenv.sh @@ -84,7 +84,7 @@ if [ "${HADOOP_EDITION}" == "1" ]; then fi fi - IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' \ + IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/'*' \ org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" ) statusCode=${?} http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java new file mode 100644 index 0000000..4069496 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop; + +/** + * Main class to compose Hadoop classpath depending on the environment. + * This class is designed to be independent on any Ignite classes as possible. + * Please make sure to pass the path separator character as the 1st parameter to the main method. + */ +public class HadoopClasspathMain { + /** + * Main method to be executed from scripts. It prints the classpath to the standard output. + * + * @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows). + */ + public static void main(String[] args) throws Exception { + if (args.length < 1) + throw new IllegalArgumentException("Path separator must be passed as the first argument."); + + String separator = args[0]; + + StringBuilder sb = new StringBuilder(); + + for (String path : HadoopClasspathUtils.classpathForProcess()) + sb.append(path).append(separator); + + System.out.println(sb); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java new file mode 100644 index 0000000..a0ef7d7 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java @@ -0,0 +1,461 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; + +/** + * Hadoop classpath utilities. + */ +public class HadoopClasspathUtils { + /** Prefix directory. */ + public static final String PREFIX = "HADOOP_PREFIX"; + + /** Hadoop home directory. */ + public static final String HOME = "HADOOP_HOME"; + + /** Hadoop common directory. */ + public static final String COMMON_HOME = "HADOOP_COMMON_HOME"; + + /** Hadoop HDFS directory. */ + public static final String HDFS_HOME = "HADOOP_HDFS_HOME"; + + /** Hadoop mapred directory. */ + public static final String MAPRED_HOME = "HADOOP_MAPRED_HOME"; + + /** Arbitrary additional dependencies. Compliant with standard Java classpath resolution. */ + public static final String HADOOP_USER_LIBS = "HADOOP_USER_LIBS"; + + /** Empty string. */ + private static final String EMPTY_STR = ""; + + /** + * Gets Hadoop class path as list of classpath elements for process. + * + * @return List of the class path elements. + * @throws IOException If failed. + */ + public static List<String> classpathForProcess() throws IOException { + List<String> res = new ArrayList<>(); + + for (final SearchDirectory dir : classpathDirectories()) { + File[] files = dir.files(); + + if (dir.useWildcard()) { + if (files.length > 0) + res.add(dir.absolutePath() + File.separator + '*'); + } + else { + for (File file : files) + res.add(file.getAbsolutePath()); + } + } + + return res; + } + + /** + * Gets Hadoop class path as a list of URLs (for in-process class loader usage). + * + * @return List of class path URLs. + * @throws IOException If failed. + */ + public static List<URL> classpathForClassLoader() throws IOException { + List<URL> res = new ArrayList<>(); + + for (SearchDirectory dir : classpathDirectories()) { + for (File file : dir.files()) { + try { + res.add(file.toURI().toURL()); + } + catch (MalformedURLException e) { + throw new IOException("Failed to convert file path to URL: " + file.getPath()); + } + } + } + + return res; + } + + /** + * Gets Hadoop locations. + * + * @return The locations as determined from the environment. + */ + public static HadoopLocations locations() throws IOException { + // Query environment. + String hadoopHome = systemOrEnv(PREFIX, systemOrEnv(HOME, EMPTY_STR)); + + String commonHome = systemOrEnv(COMMON_HOME, EMPTY_STR); + String hdfsHome = systemOrEnv(HDFS_HOME, EMPTY_STR); + String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR); + + // If any composite location is defined, use only them. + if (!isEmpty(commonHome) || !isEmpty(hdfsHome) || !isEmpty(mapredHome)) { + HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome); + + if (res.valid()) + return res; + else + throw new IOException("Failed to resolve Hadoop classpath because some environment variables are " + + "either undefined or point to nonexistent directories [" + + "[env=" + COMMON_HOME + ", value=" + commonHome + ", exists=" + res.commonExists() + "], " + + "[env=" + HDFS_HOME + ", value=" + hdfsHome + ", exists=" + res.hdfsExists() + "], " + + "[env=" + MAPRED_HOME + ", value=" + mapredHome + ", exists=" + res.mapredExists() + "]]"); + } + else if (!isEmpty(hadoopHome)) { + // All further checks will be based on HADOOP_HOME, so check for it's existence. + if (!exists(hadoopHome)) + throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " + + "variable points to nonexistent directory: " + hadoopHome); + + // Probe Apache Hadoop. + HadoopLocations res = new HadoopLocations( + hadoopHome, + hadoopHome + "/share/hadoop/common", + hadoopHome + "/share/hadoop/hdfs", + hadoopHome + "/share/hadoop/mapreduce" + ); + + if (res.valid()) + return res; + + // Probe CDH. + res = new HadoopLocations( + hadoopHome, + hadoopHome, + hadoopHome + "/../hadoop-hdfs", + hadoopHome + "/../hadoop-mapreduce" + ); + + if (res.valid()) + return res; + + // Probe HDP. + res = new HadoopLocations( + hadoopHome, + hadoopHome, + hadoopHome + "/../hadoop-hdfs-client", + hadoopHome + "/../hadoop-mapreduce-client" + ); + + if (res.valid()) + return res; + + // Failed. + throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment variable " + + "is either invalid or points to non-standard Hadoop distribution: " + hadoopHome); + } + else { + // Advise to set HADOOP_HOME only as this is preferred way to configure classpath. + throw new IOException("Failed to resolve Hadoop classpath (please define " + HOME + " environment " + + "variable and point it to your Hadoop distribution)."); + } + } + + /** + * Gets base directories to discover classpath elements in. + * + * @return Collection of directory and mask pairs. + * @throws IOException if a mandatory classpath location is not found. + */ + private static Collection<SearchDirectory> classpathDirectories() throws IOException { + HadoopLocations loc = locations(); + + Collection<SearchDirectory> res = new ArrayList<>(); + + res.add(new SearchDirectory(new File(loc.common(), "lib"), AcceptAllDirectoryFilter.INSTANCE)); + res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), AcceptAllDirectoryFilter.INSTANCE)); + res.add(new SearchDirectory(new File(loc.mapred(), "lib"), AcceptAllDirectoryFilter.INSTANCE)); + + res.add(new SearchDirectory(new File(loc.common()), new PrefixDirectoryFilter("hadoop-common-"))); + res.add(new SearchDirectory(new File(loc.common()), new PrefixDirectoryFilter("hadoop-auth-"))); + + res.add(new SearchDirectory(new File(loc.hdfs()), new PrefixDirectoryFilter("hadoop-hdfs-"))); + + res.add(new SearchDirectory(new File(loc.mapred()), + new PrefixDirectoryFilter("hadoop-mapreduce-client-common"))); + res.add(new SearchDirectory(new File(loc.mapred()), + new PrefixDirectoryFilter("hadoop-mapreduce-client-core"))); + + res.addAll(parseUserLibs()); + + return res; + } + + /** + * Parse user libs. + * + * @return Parsed libs search patterns. + * @throws IOException If failed. + */ + public static Collection<SearchDirectory> parseUserLibs() throws IOException { + return parseUserLibs(systemOrEnv(HADOOP_USER_LIBS, null)); + } + + /** + * Parse user libs. + * + * @param str String. + * @return Result. + * @throws IOException If failed. + */ + public static Collection<SearchDirectory> parseUserLibs(String str) throws IOException { + Collection<SearchDirectory> res = new LinkedList<>(); + + if (!isEmpty(str)) { + String[] tokens = normalize(str).split(File.pathSeparator); + + for (String token : tokens) { + // Skip empty tokens. + if (isEmpty(token)) + continue; + + File file = new File(token); + File dir = file.getParentFile(); + + if (token.endsWith("*")) { + assert dir != null; + + res.add(new SearchDirectory(dir, AcceptAllDirectoryFilter.INSTANCE, false)); + } + else { + // Met "/" or "C:\" pattern, nothing to do with it. + if (dir == null) + continue; + + res.add(new SearchDirectory(dir, new ExactDirectoryFilter(file.getName()), false)); + } + } + } + + return res; + } + + /** + * Get system property or environment variable with the given name. + * + * @param name Variable name. + * @param dflt Default value. + * @return Value. + */ + private static String systemOrEnv(String name, String dflt) { + String res = System.getProperty(name); + + if (res == null) + res = System.getenv(name); + + return res != null ? res : dflt; + } + + /** + * Answers if the given path denotes existing directory. + * + * @param path The directory path. + * @return {@code True} if the given path denotes an existing directory. + */ + public static boolean exists(String path) { + if (path == null) + return false; + + Path p = Paths.get(path); + + return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p); + } + + /** + * Check if string is empty. + * + * @param val Value. + * @return {@code True} if empty. + */ + private static boolean isEmpty(String val) { + return val == null || val.isEmpty(); + } + + /** + * NOramlize the string. + * + * @param str String. + * @return Normalized string. + */ + private static String normalize(String str) { + assert str != null; + + return str.trim().toLowerCase(); + } + + /** + * Simple pair-like structure to hold directory name and a mask assigned to it. + */ + public static class SearchDirectory { + /** File. */ + private final File dir; + + /** Filter. */ + private final DirectoryFilter filter; + + /** Whether directory must exist. */ + private final boolean strict; + + /** + * Constructor for directory search with strict rule. + * + * @param dir Directory. + * @param filter Filter. + * @throws IOException If failed. + */ + private SearchDirectory(File dir, DirectoryFilter filter) throws IOException { + this(dir, filter, true); + } + + /** + * Constructor. + * + * @param dir Directory. + * @param filter Filter. + * @param strict Whether directory must exist. + * @throws IOException If failed. + */ + private SearchDirectory(File dir, DirectoryFilter filter, boolean strict) throws IOException { + this.dir = dir; + this.filter = filter; + this.strict = strict; + + if (strict && !exists(dir.getAbsolutePath())) + throw new IOException("Directory cannot be read: " + dir.getAbsolutePath()); + } + + /** + * @return Absolute path. + */ + public String absolutePath() { + return dir.getAbsolutePath(); + } + + /** + * @return Child files. + */ + public File[] files() throws IOException { + File[] files = dir.listFiles(new FilenameFilter() { + @Override public boolean accept(File dir, String name) { + return filter.test(name); + } + }); + + if (files == null) { + if (strict) + throw new IOException("Failed to get directory files [dir=" + dir + ']'); + else + return new File[0]; + } + else + return files; + } + + /** + * @return {@code True} if wildcard can be used. + */ + public boolean useWildcard() { + return filter instanceof AcceptAllDirectoryFilter; + } + } + + /** + * Directory filter interface. + */ + public static interface DirectoryFilter { + /** + * Test if file with this name should be included. + * + * @param name File name. + * @return {@code True} if passed. + */ + public boolean test(String name); + } + + /** + * Filter to accept all files. + */ + public static class AcceptAllDirectoryFilter implements DirectoryFilter { + /** Singleton instance. */ + public static final AcceptAllDirectoryFilter INSTANCE = new AcceptAllDirectoryFilter(); + + /** {@inheritDoc} */ + @Override public boolean test(String name) { + return true; + } + } + + /** + * Filter which uses prefix to filter files. + */ + public static class PrefixDirectoryFilter implements DirectoryFilter { + /** Prefix. */ + private final String prefix; + + /** + * Constructor. + * + * @param prefix Prefix. + */ + public PrefixDirectoryFilter(String prefix) { + assert prefix != null; + + this.prefix = normalize(prefix); + } + + /** {@inheritDoc} */ + @Override public boolean test(String name) { + return normalize(name).startsWith(prefix); + } + } + + /** + * Filter which uses exact comparison. + */ + public static class ExactDirectoryFilter implements DirectoryFilter { + /** Name. */ + private final String name; + + /** + * Constructor. + * + * @param name Name. + */ + public ExactDirectoryFilter(String name) { + this.name = normalize(name); + } + + /** {@inheritDoc} */ + @Override public boolean test(String name) { + return normalize(name).equals(this.name); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java new file mode 100644 index 0000000..a90007f --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop; + +/** + * Simple structure to hold Hadoop directory locations. + */ +public class HadoopLocations { + /** Hadoop home. */ + private final String home; + + /** Common home. */ + private final String common; + + /** HDFS home. */ + private final String hdfs; + + /** Mapred home. */ + private final String mapred; + + /** Whether common home exists. */ + private final boolean commonExists; + + /** Whether HDFS home exists. */ + private final boolean hdfsExists; + + /** Whether mapred home exists. */ + private final boolean mapredExists; + + /** + * Constructor. + * + * @param home Hadoop home. + * @param common Common home. + * @param hdfs HDFS home. + * @param mapred Mapred home. + */ + public HadoopLocations(String home, String common, String hdfs, String mapred) { + assert common != null && hdfs != null && mapred != null; + + this.home = home; + this.common = common; + this.hdfs = hdfs; + this.mapred = mapred; + + commonExists = HadoopClasspathUtils.exists(common); + hdfsExists = HadoopClasspathUtils.exists(hdfs); + mapredExists = HadoopClasspathUtils.exists(mapred); + } + + /** + * @return Hadoop home. + */ + public String home() { + return home; + } + + /** + * @return Common home. + */ + public String common() { + return common; + } + + /** + * @return HDFS home. + */ + public String hdfs() { + return hdfs; + } + + /** + * @return Mapred home. + */ + public String mapred() { + return mapred; + } + + /** + * @return Whether common home exists. + */ + public boolean commonExists() { + return commonExists; + } + + /** + * @return Whether HDFS home exists. + */ + public boolean hdfsExists() { + return hdfsExists; + } + + /** + * @return Whether mapred home exists. + */ + public boolean mapredExists() { + return mapredExists; + } + + /** + * Whether all required directories exists. + * + * @return {@code True} if exists. + */ + public boolean valid() { + return commonExists && hdfsExists && mapredExists; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java index 2e0e271..30a6e72 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java @@ -17,44 +17,26 @@ package org.apache.ignite.internal.processors.hadoop; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.util.ClassCache; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.Nullable; +import org.jsr166.ConcurrentHashMap8; + import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import org.apache.hadoop.util.NativeCodeLoader; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon; -import org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager; -import org.apache.ignite.internal.util.ClassCache; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.jetbrains.annotations.Nullable; -import org.jsr166.ConcurrentHashMap8; -import org.objectweb.asm.AnnotationVisitor; -import org.objectweb.asm.Attribute; -import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassVisitor; -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.FieldVisitor; -import org.objectweb.asm.Handle; -import org.objectweb.asm.Label; -import org.objectweb.asm.MethodVisitor; -import org.objectweb.asm.Opcodes; -import org.objectweb.asm.Type; -import org.objectweb.asm.commons.Remapper; -import org.objectweb.asm.commons.RemappingClassAdapter; - /** * Class loader allowing explicitly load classes without delegation to parent class loader. * Also supports class parsing for finding dependencies which contain transitive dependencies @@ -66,8 +48,21 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { registerAsParallelCapable(); } - /** Name of the Hadoop daemon class. */ - public static final String HADOOP_DAEMON_CLASS_NAME = "org.apache.hadoop.util.Daemon"; + /** Hadoop class name: Daemon. */ + public static final String CLS_DAEMON = "org.apache.hadoop.util.Daemon"; + + /** Hadoop class name: ShutdownHookManager. */ + public static final String CLS_SHUTDOWN_HOOK_MANAGER = "org.apache.hadoop.util.ShutdownHookManager"; + + /** Hadoop class name: NativeCodeLoader. */ + public static final String CLS_NATIVE_CODE_LOADER = "org.apache.hadoop.util.NativeCodeLoader"; + + /** Hadoop class name: Daemon replacement. */ + public static final String CLS_DAEMON_REPLACE = "org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon"; + + /** Hadoop class name: ShutdownHookManager replacement. */ + public static final String CLS_SHUTDOWN_HOOK_MANAGER_REPLACE = + "org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager"; /** Name of libhadoop library. */ private static final String LIBHADOOP = "hadoop."; @@ -82,9 +77,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { private static volatile Collection<URL> hadoopJars; /** */ - private static final Map<String, Boolean> cache = new ConcurrentHashMap8<>(); - - /** */ private static final Map<String, byte[]> bytesCache = new ConcurrentHashMap8<>(); /** Class cache. */ @@ -152,10 +144,12 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { private void initializeNativeLibraries() { try { // This must trigger native library load. - Class.forName(NativeCodeLoader.class.getName(), true, APP_CLS_LDR); + // TODO: Do not delegate to APP LDR + Class.forName(CLS_NATIVE_CODE_LOADER, true, APP_CLS_LDR); final Vector<Object> curVector = U.field(this, "nativeLibraries"); + // TODO: Do not delegate to APP LDR ClassLoader ldr = APP_CLS_LDR; while (ldr != null) { @@ -192,55 +186,24 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { } } - /** - * Need to parse only Ignite Hadoop and IGFS classes. - * - * @param cls Class name. - * @return {@code true} if we need to check this class. - */ - private static boolean isHadoopIgfs(String cls) { - String ignitePkgPrefix = "org.apache.ignite"; - - int len = ignitePkgPrefix.length(); - - return cls.startsWith(ignitePkgPrefix) && ( - cls.indexOf("igfs.", len) != -1 || - cls.indexOf(".fs.", len) != -1 || - cls.indexOf("hadoop.", len) != -1); - } - - /** - * @param cls Class name. - * @return {@code true} If this is Hadoop class. - */ - private static boolean isHadoop(String cls) { - return cls.startsWith("org.apache.hadoop."); - } - /** {@inheritDoc} */ @Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException { try { - if (isHadoop(name)) { // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath. - if (name.endsWith(".util.ShutdownHookManager")) // Dirty hack to get rid of Hadoop shutdown hooks. - return loadFromBytes(name, HadoopShutdownHookManager.class.getName()); - else if (name.equals(HADOOP_DAEMON_CLASS_NAME)) + // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath. + if (HadoopClassLoaderUtils.isHadoop(name)) { + if (name.equals(CLS_SHUTDOWN_HOOK_MANAGER)) // Dirty hack to get rid of Hadoop shutdown hooks. + return loadReplace(name, CLS_SHUTDOWN_HOOK_MANAGER_REPLACE); + else if (name.equals(CLS_DAEMON)) // We replace this in order to be able to forcibly stop some daemon threads // that otherwise never stop (e.g. PeerCache runnables): - return loadFromBytes(name, HadoopDaemon.class.getName()); + return loadReplace(name, CLS_DAEMON_REPLACE); return loadClassExplicitly(name, resolve); } - if (isHadoopIgfs(name)) { // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop. - Boolean hasDeps = cache.get(name); - - if (hasDeps == null) { - hasDeps = hasExternalDependencies(name); - - cache.put(name, hasDeps); - } - - if (hasDeps) + // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop. + if (HadoopClassLoaderUtils.isHadoopIgfs(name)) { + if (hasExternalDependencies(name)) return loadClassExplicitly(name, resolve); } @@ -252,55 +215,31 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { } /** - * @param name Name. - * @param replace Replacement. + * Load a class replacing it with our own implementation. + * + * @param originalName Name. + * @param replaceName Replacement. * @return Class. */ - private Class<?> loadFromBytes(final String name, final String replace) { - synchronized (getClassLoadingLock(name)) { + private Class<?> loadReplace(final String originalName, final String replaceName) { + synchronized (getClassLoadingLock(originalName)) { // First, check if the class has already been loaded - Class c = findLoadedClass(name); + Class c = findLoadedClass(originalName); if (c != null) return c; - byte[] bytes = bytesCache.get(name); + byte[] bytes = bytesCache.get(originalName); if (bytes == null) { - InputStream in = loadClassBytes(getParent(), replace); - - ClassReader rdr; - - try { - rdr = new ClassReader(in); - } - catch (IOException e) { - throw new RuntimeException(e); - } - - ClassWriter w = new ClassWriter(Opcodes.ASM4); - - rdr.accept(new RemappingClassAdapter(w, new Remapper() { - /** */ - String replaceType = replace.replace('.', '/'); + InputStream in = HadoopClassLoaderUtils.loadClassBytes(getParent(), replaceName); - /** */ - String nameType = name.replace('.', '/'); + bytes = HadoopClassLoaderUtils.loadReplace(in, originalName, replaceName); - @Override public String map(String type) { - if (type.equals(replaceType)) - return nameType; - - return type; - } - }), ClassReader.EXPAND_FRAMES); - - bytes = w.toByteArray(); - - bytesCache.put(name, bytes); + bytesCache.put(originalName, bytes); } - return defineClass(name, bytes, 0, bytes.length); + return defineClass(originalName, bytes, 0, bytes.length); } } @@ -347,111 +286,13 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { } /** - * @param ldr Loader. - * @param clsName Class. - * @return Input stream. - */ - @Nullable private InputStream loadClassBytes(ClassLoader ldr, String clsName) { - return ldr.getResourceAsStream(clsName.replace('.', '/') + ".class"); - } - - /** * Check whether class has external dependencies on Hadoop. * * @param clsName Class name. * @return {@code True} if class has external dependencies. */ boolean hasExternalDependencies(String clsName) { - CollectingContext ctx = new CollectingContext(); - - ctx.annVisitor = new CollectingAnnotationVisitor(ctx); - ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor); - ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor); - ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor); - - return hasExternalDependencies(clsName, ctx); - } - - /** - * Check whether class has external dependencies on Hadoop. - * - * @param clsName Class name. - * @param ctx Context. - * @return {@code true} If the class has external dependencies. - */ - boolean hasExternalDependencies(String clsName, CollectingContext ctx) { - if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external. - return true; - - // Try to get from parent to check if the type accessible. - InputStream in = loadClassBytes(getParent(), clsName); - - if (in == null) // The class is external itself, it must be loaded from this class loader. - return true; - - if (!isHadoopIgfs(clsName)) // Other classes should not have external dependencies. - return false; - - final ClassReader rdr; - - try { - rdr = new ClassReader(in); - } - catch (IOException e) { - throw new RuntimeException("Failed to read class: " + clsName, e); - } - - ctx.visited.add(clsName); - - rdr.accept(ctx.clsVisitor, 0); - - if (ctx.found) // We already know that we have dependencies, no need to check parent. - return true; - - // Here we are known to not have any dependencies but possibly we have a parent which has them. - int idx = clsName.lastIndexOf('$'); - - if (idx == -1) // No parent class. - return false; - - String parentCls = clsName.substring(0, idx); - - if (ctx.visited.contains(parentCls)) - return false; - - Boolean res = cache.get(parentCls); - - if (res == null) - res = hasExternalDependencies(parentCls, ctx); - - return res; - } - - /** - * @param name Class name. - * @return {@code true} If this is a valid class name. - */ - private static boolean validateClassName(String name) { - int len = name.length(); - - if (len <= 1) - return false; - - if (!Character.isJavaIdentifierStart(name.charAt(0))) - return false; - - boolean hasDot = false; - - for (int i = 1; i < len; i++) { - char c = name.charAt(i); - - if (c == '.') - hasDot = true; - else if (!Character.isJavaIdentifierPart(c)) - return false; - } - - return hasDot; + return HadoopClassLoaderUtils.hasExternalDependencies(clsName, getParent()); } /** @@ -519,446 +360,4 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { public String name() { return name; } - - /** - * Context for dependencies collection. - */ - private class CollectingContext { - /** Visited classes. */ - private final Set<String> visited = new HashSet<>(); - - /** Whether dependency found. */ - private boolean found; - - /** Annotation visitor. */ - private AnnotationVisitor annVisitor; - - /** Method visitor. */ - private MethodVisitor mthdVisitor; - - /** Field visitor. */ - private FieldVisitor fldVisitor; - - /** Class visitor. */ - private ClassVisitor clsVisitor; - - /** - * Processes a method descriptor - * @param methDesc The method desc String. - */ - void onMethodsDesc(final String methDesc) { - // Process method return type: - onType(Type.getReturnType(methDesc)); - - if (found) - return; - - // Process method argument types: - for (Type t: Type.getArgumentTypes(methDesc)) { - onType(t); - - if (found) - return; - } - } - - /** - * Processes dependencies of a class. - * - * @param depCls The class name as dot-notated FQN. - */ - void onClass(final String depCls) { - assert depCls.indexOf('/') == -1 : depCls; // class name should be fully converted to dot notation. - assert depCls.charAt(0) != 'L' : depCls; - assert validateClassName(depCls) : depCls; - - if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // Filter out platform classes. - return; - - if (visited.contains(depCls)) - return; - - Boolean res = cache.get(depCls); - - if (res == Boolean.TRUE || (res == null && hasExternalDependencies(depCls, this))) - found = true; - } - - /** - * Analyses dependencies of given type. - * - * @param t The type to process. - */ - void onType(Type t) { - if (t == null) - return; - - int sort = t.getSort(); - - switch (sort) { - case Type.ARRAY: - onType(t.getElementType()); - - break; - - case Type.OBJECT: - onClass(t.getClassName()); - - break; - } - } - - /** - * Analyses dependencies of given object type. - * - * @param objType The object type to process. - */ - void onInternalTypeName(String objType) { - if (objType == null) - return; - - assert objType.length() > 1 : objType; - - if (objType.charAt(0) == '[') - // handle array. In this case this is a type descriptor notation, like "[Ljava/lang/Object;" - onType(objType); - else { - assert objType.indexOf('.') == -1 : objType; // Must be slash-separated FQN. - - String clsName = objType.replace('/', '.'); // Convert it to dot notation. - - onClass(clsName); // Process. - } - } - - /** - * Type description analyser. - * - * @param desc The description. - */ - void onType(String desc) { - if (!F.isEmpty(desc)) { - if (desc.length() <= 1) - return; // Optimization: filter out primitive types in early stage. - - Type t = Type.getType(desc); - - onType(t); - } - } - } - - /** - * Annotation visitor. - */ - private static class CollectingAnnotationVisitor extends AnnotationVisitor { - /** */ - final CollectingContext ctx; - - /** - * Annotation visitor. - * - * @param ctx The collector. - */ - CollectingAnnotationVisitor(CollectingContext ctx) { - super(Opcodes.ASM4); - - this.ctx = ctx; - } - - /** {@inheritDoc} */ - @Override public AnnotationVisitor visitAnnotation(String name, String desc) { - if (ctx.found) - return null; - - ctx.onType(desc); - - return this; - } - - /** {@inheritDoc} */ - @Override public void visitEnum(String name, String desc, String val) { - if (ctx.found) - return; - - ctx.onType(desc); - } - - /** {@inheritDoc} */ - @Override public AnnotationVisitor visitArray(String name) { - return ctx.found ? null : this; - } - - /** {@inheritDoc} */ - @Override public void visit(String name, Object val) { - if (ctx.found) - return; - - if (val instanceof Type) - ctx.onType((Type)val); - } - - /** {@inheritDoc} */ - @Override public void visitEnd() { - // No-op. - } - } - - /** - * Field visitor. - */ - private static class CollectingFieldVisitor extends FieldVisitor { - /** Collector. */ - private final CollectingContext ctx; - - /** Annotation visitor. */ - private final AnnotationVisitor av; - - /** - * Constructor. - */ - CollectingFieldVisitor(CollectingContext ctx, AnnotationVisitor av) { - super(Opcodes.ASM4); - - this.ctx = ctx; - this.av = av; - } - - /** {@inheritDoc} */ - @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { - if (ctx.found) - return null; - - ctx.onType(desc); - - return ctx.found ? null : av; - } - - /** {@inheritDoc} */ - @Override public void visitAttribute(Attribute attr) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void visitEnd() { - // No-op. - } - } - - /** - * Class visitor. - */ - private static class CollectingClassVisitor extends ClassVisitor { - /** Collector. */ - private final CollectingContext ctx; - - /** Annotation visitor. */ - private final AnnotationVisitor av; - - /** Method visitor. */ - private final MethodVisitor mv; - - /** Field visitor. */ - private final FieldVisitor fv; - - /** - * Constructor. - * - * @param ctx Collector. - * @param av Annotation visitor. - * @param mv Method visitor. - * @param fv Field visitor. - */ - CollectingClassVisitor(CollectingContext ctx, AnnotationVisitor av, MethodVisitor mv, FieldVisitor fv) { - super(Opcodes.ASM4); - - this.ctx = ctx; - this.av = av; - this.mv = mv; - this.fv = fv; - } - - /** {@inheritDoc} */ - @Override public void visit(int i, int i2, String name, String signature, String superName, String[] ifaces) { - if (ctx.found) - return; - - ctx.onInternalTypeName(superName); - - if (ctx.found) - return; - - if (ifaces != null) { - for (String iface : ifaces) { - ctx.onInternalTypeName(iface); - - if (ctx.found) - return; - } - } - } - - /** {@inheritDoc} */ - @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { - if (ctx.found) - return null; - - ctx.onType(desc); - - return ctx.found ? null : av; - } - - /** {@inheritDoc} */ - @Override public void visitInnerClass(String name, String outerName, String innerName, int i) { - if (ctx.found) - return; - - ctx.onInternalTypeName(name); - } - - /** {@inheritDoc} */ - @Override public FieldVisitor visitField(int i, String name, String desc, String signature, Object val) { - if (ctx.found) - return null; - - ctx.onType(desc); - - return ctx.found ? null : fv; - } - - /** {@inheritDoc} */ - @Override public MethodVisitor visitMethod(int i, String name, String desc, String signature, - String[] exceptions) { - if (ctx.found) - return null; - - ctx.onMethodsDesc(desc); - - // Process declared method exceptions: - if (exceptions != null) { - for (String e : exceptions) - ctx.onInternalTypeName(e); - } - - return ctx.found ? null : mv; - } - } - - /** - * Method visitor. - */ - private static class CollectingMethodVisitor extends MethodVisitor { - /** Collector. */ - private final CollectingContext ctx; - - /** Annotation visitor. */ - private final AnnotationVisitor av; - - /** - * Constructor. - * - * @param ctx Collector. - * @param av Annotation visitor. - */ - private CollectingMethodVisitor(CollectingContext ctx, AnnotationVisitor av) { - super(Opcodes.ASM4); - - this.ctx = ctx; - this.av = av; - } - - /** {@inheritDoc} */ - @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { - if (ctx.found) - return null; - - ctx.onType(desc); - - return ctx.found ? null : av; - } - - /** {@inheritDoc} */ - @Override public AnnotationVisitor visitParameterAnnotation(int i, String desc, boolean b) { - if (ctx.found) - return null; - - ctx.onType(desc); - - return ctx.found ? null : av; - } - - /** {@inheritDoc} */ - @Override public AnnotationVisitor visitAnnotationDefault() { - return ctx.found ? null : av; - } - - /** {@inheritDoc} */ - @Override public void visitFieldInsn(int opcode, String owner, String name, String desc) { - if (ctx.found) - return; - - ctx.onInternalTypeName(owner); - - if (ctx.found) - return; - - ctx.onType(desc); - } - - /** {@inheritDoc} */ - @Override public void visitInvokeDynamicInsn(String name, String desc, Handle bsm, Object... bsmArgs) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void visitFrame(int type, int nLoc, Object[] locTypes, int nStack, Object[] stackTypes) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void visitLocalVariable(String name, String desc, String signature, Label lb, - Label lb2, int i) { - if (ctx.found) - return; - - ctx.onType(desc); - } - - /** {@inheritDoc} */ - @Override public void visitMethodInsn(int i, String owner, String name, String desc) { - if (ctx.found) - return; - - ctx.onInternalTypeName(owner); - - if (ctx.found) - return; - - ctx.onMethodsDesc(desc); - } - - /** {@inheritDoc} */ - @Override public void visitMultiANewArrayInsn(String desc, int dim) { - if (ctx.found) - return; - - ctx.onType(desc); - } - - /** {@inheritDoc} */ - @Override public void visitTryCatchBlock(Label start, Label end, Label hndl, String typeStr) { - if (ctx.found) - return; - - ctx.onInternalTypeName(typeStr); - } - - /** {@inheritDoc} */ - @Override public void visitTypeInsn(int opcode, String type) { - if (ctx.found) - return; - - ctx.onInternalTypeName(type); - } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java new file mode 100644 index 0000000..3415d6a --- /dev/null +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java @@ -0,0 +1,684 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop; + +import org.apache.ignite.internal.util.typedef.F; +import org.jetbrains.annotations.Nullable; +import org.jsr166.ConcurrentHashMap8; +import org.objectweb.asm.AnnotationVisitor; +import org.objectweb.asm.Attribute; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; +import org.objectweb.asm.commons.Remapper; +import org.objectweb.asm.commons.RemappingClassAdapter; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Utility methods for Hadoop classloader required to avoid direct 3rd-party dependencies in class loader. + */ +public class HadoopClassLoaderUtils { + /** Cache for resolved dependency info. */ + private static final Map<String, Boolean> dependenciesCache = new ConcurrentHashMap8<>(); + + /** + * Load special replacement and impersonate + * + * @param in Input stream. + * @param originalName Original class name. + * @param replaceName Replacer class name. + * @return Result. + */ + public static byte[] loadReplace(InputStream in, final String originalName, final String replaceName) { + ClassReader rdr; + + try { + rdr = new ClassReader(in); + } + catch (IOException e) { + throw new RuntimeException(e); + } + + ClassWriter w = new ClassWriter(Opcodes.ASM4); + + rdr.accept(new RemappingClassAdapter(w, new Remapper() { + /** */ + String replaceType = replaceName.replace('.', '/'); + + /** */ + String nameType = originalName.replace('.', '/'); + + @Override public String map(String type) { + if (type.equals(replaceType)) + return nameType; + + return type; + } + }), ClassReader.EXPAND_FRAMES); + + return w.toByteArray(); + } + + /** + * @param cls Class name. + * @return {@code true} If this is Hadoop class. + */ + public static boolean isHadoop(String cls) { + return cls.startsWith("org.apache.hadoop."); + } + + /** + * Need to parse only Ignite Hadoop and IGFS classes. + * + * @param cls Class name. + * @return {@code true} if we need to check this class. + */ + public static boolean isHadoopIgfs(String cls) { + String ignitePkgPrefix = "org.apache.ignite"; + + int len = ignitePkgPrefix.length(); + + return cls.startsWith(ignitePkgPrefix) && ( + cls.indexOf("igfs.", len) != -1 || + cls.indexOf(".fs.", len) != -1 || + cls.indexOf("hadoop.", len) != -1); + } + + /** + * @param ldr Loader. + * @param clsName Class. + * @return Input stream. + */ + @Nullable public static InputStream loadClassBytes(ClassLoader ldr, String clsName) { + return ldr.getResourceAsStream(clsName.replace('.', '/') + ".class"); + } + + /** + * Check whether class has external dependencies on Hadoop. + * + * @param clsName Class name. + * @param parentClsLdr Parent class loader. + * @return {@code True} if class has external dependencies. + */ + static boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr) { + Boolean hasDeps = dependenciesCache.get(clsName); + + if (hasDeps == null) { + CollectingContext ctx = new CollectingContext(parentClsLdr); + + ctx.annVisitor = new CollectingAnnotationVisitor(ctx); + ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor); + ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor); + ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor); + + hasDeps = hasExternalDependencies(clsName, parentClsLdr, ctx); + + dependenciesCache.put(clsName, hasDeps); + } + + return hasDeps; + } + + /** + * Check whether class has external dependencies on Hadoop. + * + * @param clsName Class name. + * @param parentClsLdr Parent class loader. + * @param ctx Context. + * @return {@code true} If the class has external dependencies. + */ + static boolean hasExternalDependencies(String clsName, ClassLoader parentClsLdr, CollectingContext ctx) { + if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external. + return true; + + // Try to get from parent to check if the type accessible. + InputStream in = loadClassBytes(parentClsLdr, clsName); + + if (in == null) // The class is external itself, it must be loaded from this class loader. + return true; + + if (!isHadoopIgfs(clsName)) // Other classes should not have external dependencies. + return false; + + final ClassReader rdr; + + try { + rdr = new ClassReader(in); + } + catch (IOException e) { + throw new RuntimeException("Failed to read class: " + clsName, e); + } + + ctx.visited.add(clsName); + + rdr.accept(ctx.clsVisitor, 0); + + if (ctx.found) // We already know that we have dependencies, no need to check parent. + return true; + + // Here we are known to not have any dependencies but possibly we have a parent which has them. + int idx = clsName.lastIndexOf('$'); + + if (idx == -1) // No parent class. + return false; + + String parentCls = clsName.substring(0, idx); + + if (ctx.visited.contains(parentCls)) + return false; + + Boolean res = dependenciesCache.get(parentCls); + + if (res == null) + res = hasExternalDependencies(parentCls, parentClsLdr, ctx); + + return res; + } + + /** + * @param name Class name. + * @return {@code true} If this is a valid class name. + */ + private static boolean validateClassName(String name) { + int len = name.length(); + + if (len <= 1) + return false; + + if (!Character.isJavaIdentifierStart(name.charAt(0))) + return false; + + boolean hasDot = false; + + for (int i = 1; i < len; i++) { + char c = name.charAt(i); + + if (c == '.') + hasDot = true; + else if (!Character.isJavaIdentifierPart(c)) + return false; + } + + return hasDot; + } + + /** + * Context for dependencies collection. + */ + private static class CollectingContext { + /** Visited classes. */ + private final Set<String> visited = new HashSet<>(); + + /** Parent class loader. */ + private final ClassLoader parentClsLdr; + + /** Whether dependency found. */ + private boolean found; + + /** Annotation visitor. */ + private AnnotationVisitor annVisitor; + + /** Method visitor. */ + private MethodVisitor mthdVisitor; + + /** Field visitor. */ + private FieldVisitor fldVisitor; + + /** Class visitor. */ + private ClassVisitor clsVisitor; + + /** + * Constrcutor. + * + * @param parentClsLdr Parent class loader. + */ + private CollectingContext(ClassLoader parentClsLdr) { + this.parentClsLdr = parentClsLdr; + } + + /** + * Processes a method descriptor + * @param methDesc The method desc String. + */ + void onMethodsDesc(final String methDesc) { + // Process method return type: + onType(Type.getReturnType(methDesc)); + + if (found) + return; + + // Process method argument types: + for (Type t: Type.getArgumentTypes(methDesc)) { + onType(t); + + if (found) + return; + } + } + + /** + * Processes dependencies of a class. + * + * @param depCls The class name as dot-notated FQN. + */ + void onClass(final String depCls) { + assert depCls.indexOf('/') == -1 : depCls; // class name should be fully converted to dot notation. + assert depCls.charAt(0) != 'L' : depCls; + assert validateClassName(depCls) : depCls; + + if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // Filter out platform classes. + return; + + if (visited.contains(depCls)) + return; + + Boolean res = dependenciesCache.get(depCls); + + if (res == Boolean.TRUE || (res == null && hasExternalDependencies(depCls, parentClsLdr, this))) + found = true; + } + + /** + * Analyses dependencies of given type. + * + * @param t The type to process. + */ + void onType(Type t) { + if (t == null) + return; + + int sort = t.getSort(); + + switch (sort) { + case Type.ARRAY: + onType(t.getElementType()); + + break; + + case Type.OBJECT: + onClass(t.getClassName()); + + break; + } + } + + /** + * Analyses dependencies of given object type. + * + * @param objType The object type to process. + */ + void onInternalTypeName(String objType) { + if (objType == null) + return; + + assert objType.length() > 1 : objType; + + if (objType.charAt(0) == '[') + // handle array. In this case this is a type descriptor notation, like "[Ljava/lang/Object;" + onType(objType); + else { + assert objType.indexOf('.') == -1 : objType; // Must be slash-separated FQN. + + String clsName = objType.replace('/', '.'); // Convert it to dot notation. + + onClass(clsName); // Process. + } + } + + /** + * Type description analyser. + * + * @param desc The description. + */ + void onType(String desc) { + if (!F.isEmpty(desc)) { + if (desc.length() <= 1) + return; // Optimization: filter out primitive types in early stage. + + Type t = Type.getType(desc); + + onType(t); + } + } + } + + /** + * Annotation visitor. + */ + private static class CollectingAnnotationVisitor extends AnnotationVisitor { + /** */ + final CollectingContext ctx; + + /** + * Annotation visitor. + * + * @param ctx The collector. + */ + CollectingAnnotationVisitor(CollectingContext ctx) { + super(Opcodes.ASM4); + + this.ctx = ctx; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotation(String name, String desc) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return this; + } + + /** {@inheritDoc} */ + @Override public void visitEnum(String name, String desc, String val) { + if (ctx.found) + return; + + ctx.onType(desc); + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitArray(String name) { + return ctx.found ? null : this; + } + + /** {@inheritDoc} */ + @Override public void visit(String name, Object val) { + if (ctx.found) + return; + + if (val instanceof Type) + ctx.onType((Type)val); + } + + /** {@inheritDoc} */ + @Override public void visitEnd() { + // No-op. + } + } + + /** + * Field visitor. + */ + private static class CollectingFieldVisitor extends FieldVisitor { + /** Collector. */ + private final CollectingContext ctx; + + /** Annotation visitor. */ + private final AnnotationVisitor av; + + /** + * Constructor. + */ + CollectingFieldVisitor(CollectingContext ctx, AnnotationVisitor av) { + super(Opcodes.ASM4); + + this.ctx = ctx; + this.av = av; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public void visitAttribute(Attribute attr) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void visitEnd() { + // No-op. + } + } + + /** + * Class visitor. + */ + private static class CollectingClassVisitor extends ClassVisitor { + /** Collector. */ + private final CollectingContext ctx; + + /** Annotation visitor. */ + private final AnnotationVisitor av; + + /** Method visitor. */ + private final MethodVisitor mv; + + /** Field visitor. */ + private final FieldVisitor fv; + + /** + * Constructor. + * + * @param ctx Collector. + * @param av Annotation visitor. + * @param mv Method visitor. + * @param fv Field visitor. + */ + CollectingClassVisitor(CollectingContext ctx, AnnotationVisitor av, MethodVisitor mv, FieldVisitor fv) { + super(Opcodes.ASM4); + + this.ctx = ctx; + this.av = av; + this.mv = mv; + this.fv = fv; + } + + /** {@inheritDoc} */ + @Override public void visit(int i, int i2, String name, String signature, String superName, String[] ifaces) { + if (ctx.found) + return; + + ctx.onInternalTypeName(superName); + + if (ctx.found) + return; + + if (ifaces != null) { + for (String iface : ifaces) { + ctx.onInternalTypeName(iface); + + if (ctx.found) + return; + } + } + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public void visitInnerClass(String name, String outerName, String innerName, int i) { + if (ctx.found) + return; + + ctx.onInternalTypeName(name); + } + + /** {@inheritDoc} */ + @Override public FieldVisitor visitField(int i, String name, String desc, String signature, Object val) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : fv; + } + + /** {@inheritDoc} */ + @Override public MethodVisitor visitMethod(int i, String name, String desc, String signature, + String[] exceptions) { + if (ctx.found) + return null; + + ctx.onMethodsDesc(desc); + + // Process declared method exceptions: + if (exceptions != null) { + for (String e : exceptions) + ctx.onInternalTypeName(e); + } + + return ctx.found ? null : mv; + } + } + + /** + * Method visitor. + */ + private static class CollectingMethodVisitor extends MethodVisitor { + /** Collector. */ + private final CollectingContext ctx; + + /** Annotation visitor. */ + private final AnnotationVisitor av; + + /** + * Constructor. + * + * @param ctx Collector. + * @param av Annotation visitor. + */ + private CollectingMethodVisitor(CollectingContext ctx, AnnotationVisitor av) { + super(Opcodes.ASM4); + + this.ctx = ctx; + this.av = av; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitParameterAnnotation(int i, String desc, boolean b) { + if (ctx.found) + return null; + + ctx.onType(desc); + + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public AnnotationVisitor visitAnnotationDefault() { + return ctx.found ? null : av; + } + + /** {@inheritDoc} */ + @Override public void visitFieldInsn(int opcode, String owner, String name, String desc) { + if (ctx.found) + return; + + ctx.onInternalTypeName(owner); + + if (ctx.found) + return; + + ctx.onType(desc); + } + + /** {@inheritDoc} */ + @Override public void visitInvokeDynamicInsn(String name, String desc, Handle bsm, Object... bsmArgs) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void visitFrame(int type, int nLoc, Object[] locTypes, int nStack, Object[] stackTypes) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void visitLocalVariable(String name, String desc, String signature, Label lb, + Label lb2, int i) { + if (ctx.found) + return; + + ctx.onType(desc); + } + + /** {@inheritDoc} */ + @Override public void visitMethodInsn(int i, String owner, String name, String desc) { + if (ctx.found) + return; + + ctx.onInternalTypeName(owner); + + if (ctx.found) + return; + + ctx.onMethodsDesc(desc); + } + + /** {@inheritDoc} */ + @Override public void visitMultiANewArrayInsn(String desc, int dim) { + if (ctx.found) + return; + + ctx.onType(desc); + } + + /** {@inheritDoc} */ + @Override public void visitTryCatchBlock(Label start, Label end, Label hndl, String typeStr) { + if (ctx.found) + return; + + ctx.onInternalTypeName(typeStr); + } + + /** {@inheritDoc} */ + @Override public void visitTypeInsn(int opcode, String type) { + if (ctx.found) + return; + + ctx.onInternalTypeName(type); + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/2fe0272b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java deleted file mode 100644 index 4069496..0000000 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -/** - * Main class to compose Hadoop classpath depending on the environment. - * This class is designed to be independent on any Ignite classes as possible. - * Please make sure to pass the path separator character as the 1st parameter to the main method. - */ -public class HadoopClasspathMain { - /** - * Main method to be executed from scripts. It prints the classpath to the standard output. - * - * @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows). - */ - public static void main(String[] args) throws Exception { - if (args.length < 1) - throw new IllegalArgumentException("Path separator must be passed as the first argument."); - - String separator = args[0]; - - StringBuilder sb = new StringBuilder(); - - for (String path : HadoopClasspathUtils.classpathForProcess()) - sb.append(path).append(separator); - - System.out.println(sb); - } -}