IGNITE-3185: Hadoop: Improved classpath resolution logic.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/34798d3a Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/34798d3a Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/34798d3a Branch: refs/heads/ignite-1232 Commit: 34798d3ae007c99c4c813d7a1e060f4f145b4ec2 Parents: 9d033e5 Author: vozerov-gridgain <[email protected]> Authored: Fri Jul 1 15:39:40 2016 +0300 Committer: vozerov-gridgain <[email protected]> Committed: Wed Jul 6 09:54:18 2016 +0300 ---------------------------------------------------------------------- .../processors/hadoop/HadoopClasspathUtils.java | 160 +++++++++++++------ .../processors/hadoop/HadoopLocations.java | 79 +++++++-- .../processors/hadoop/HadoopProcessor.java | 12 +- 3 files changed, 179 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/34798d3a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java index 986b23e..121fcab 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java @@ -33,6 +33,24 @@ import java.util.List; * Hadoop classpath utilities. */ public class HadoopClasspathUtils { + /** Prefix directory. */ + public static final String PREFIX = "HADOOP_PREFIX"; + + /** Home directory. */ + public static final String HOME = "HADOOP_HOME"; + + /** Home directory. */ + public static final String COMMON_HOME = "HADOOP_COMMON_HOME"; + + /** Home directory. */ + public static final String HDFS_HOME = "HADOOP_HDFS_HOME"; + + /** Home directory. */ + public static final String MAPRED_HOME = "HADOOP_MAPRED_HOME"; + + /** Empty string. */ + private static final String EMPTY_STR = ""; + /** * Gets Hadoop class path as list of classpath elements for process. * @@ -80,16 +98,77 @@ public class HadoopClasspathUtils { /** * Gets Hadoop locations. * - * @return The Hadoop locations, never null. + * @return The locations as determined from the environment. */ - public static HadoopLocations hadoopLocations() throws IOException { - final String hadoopHome = systemOrEnv("HADOOP_HOME", systemOrEnv("HADOOP_PREFIX", null)); + public static HadoopLocations locations() throws IOException { + // Query environment. + String hadoopHome = systemOrEnv(PREFIX, systemOrEnv(HOME, EMPTY_STR)); + + String commonHome = systemOrEnv(COMMON_HOME, EMPTY_STR); + String hdfsHome = systemOrEnv(HDFS_HOME, EMPTY_STR); + String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR); - String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common"); - String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs"); - String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce"); + // If any composite location is defined, use only them. + if (!isEmpty(commonHome) || !isEmpty(hdfsHome) || !isEmpty(mapredHome)) { + HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome); - return new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome); + if (res.valid()) + return res; + else + throw new IOException("Failed to resolve Hadoop classpath because some environment variables are " + + "either undefined or point to nonexistent directories [" + + "[env=" + COMMON_HOME + ", value=" + commonHome + ", exists=" + res.commonExists() + "], " + + "[env=" + HDFS_HOME + ", value=" + hdfsHome + ", exists=" + res.hdfsExists() + "], " + + "[env=" + MAPRED_HOME + ", value=" + mapredHome + ", exists=" + res.mapredExists() + "]]"); + } + else if (!isEmpty(hadoopHome)) { + // All further checks will be based on HADOOP_HOME, so check for it's existence. + if (!exists(hadoopHome)) + throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " + + "variable points to nonexistent directory: " + hadoopHome); + + // Probe Apache Hadoop. + HadoopLocations res = new HadoopLocations( + hadoopHome, + hadoopHome + "/share/hadoop/common", + hadoopHome + "/share/hadoop/hdfs", + hadoopHome + "/share/hadoop/mapreduce" + ); + + if (res.valid()) + return res; + + // Probe CDH. + res = new HadoopLocations( + hadoopHome, + hadoopHome, + hadoopHome + "/../hadoop-hdfs", + hadoopHome + "/../hadoop-mapreduce" + ); + + if (res.valid()) + return res; + + // Probe HDP. + res = new HadoopLocations( + hadoopHome, + hadoopHome, + hadoopHome + "/../hadoop-hdfs-client", + hadoopHome + "/../hadoop-mapreduce-client" + ); + + if (res.valid()) + return res; + + // Failed. + throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment variable " + + "is either invalid or points to non-standard Hadoop distribution: " + hadoopHome); + } + else { + // Advise to set HADOOP_HOME only as this is preferred way to configure classpath. + throw new IOException("Failed to resolve Hadoop classpath (please define " + HOME + " environment " + + "variable and point it to your Hadoop distribution)."); + } } /** @@ -99,57 +178,30 @@ public class HadoopClasspathUtils { * @throws IOException if a mandatory classpath location is not found. */ private static Collection<SearchDirectory> classpathDirectories() throws IOException { - HadoopLocations loc = hadoopLocations(); + HadoopLocations loc = locations(); Collection<SearchDirectory> res = new ArrayList<>(); - res.add(new SearchDirectory(new File(loc.commonHome(), "lib"), null)); - res.add(new SearchDirectory(new File(loc.hdfsHome(), "lib"), null)); - res.add(new SearchDirectory(new File(loc.mapredHome(), "lib"), null)); + res.add(new SearchDirectory(new File(loc.common(), "lib"), null)); + res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), null)); + res.add(new SearchDirectory(new File(loc.mapred(), "lib"), null)); - res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-common-")); - res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-auth-")); + res.add(new SearchDirectory(new File(loc.common()), "hadoop-common-")); + res.add(new SearchDirectory(new File(loc.common()), "hadoop-auth-")); - res.add(new SearchDirectory(new File(loc.hdfsHome()), "hadoop-hdfs-")); + res.add(new SearchDirectory(new File(loc.hdfs()), "hadoop-hdfs-")); - res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-common")); - res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-core")); + res.add(new SearchDirectory(new File(loc.mapred()), "hadoop-mapreduce-client-common")); + res.add(new SearchDirectory(new File(loc.mapred()), "hadoop-mapreduce-client-core")); return res; } /** - * Resolves a Hadoop location directory. - * - * @param envVarName Environment variable name. The value denotes the location path. - * @param hadoopHome Hadoop home location, may be null. - * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator. - * @throws IOException If the value cannot be resolved to an existing directory. - */ - private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath) - throws IOException { - String val = systemOrEnv(envVarName, null); - - if (val == null) { - // The env. variable is not set. Try to resolve the location relative HADOOP_HOME: - if (!directoryExists(hadoopHome)) - throw new IOException("Failed to resolve Hadoop installation location. " + - envVarName + " or HADOOP_HOME environment variable should be set."); - - val = hadoopHome + expHadoopHomeRelativePath; - } - - if (!directoryExists(val)) - throw new IOException("Failed to resolve Hadoop location [path=" + val + ']'); - - return val; - } - - /** - * Note that this method does not treat empty value as an absent value. + * Get system property or environment variable with the given name. * * @param name Variable name. - * @param dflt Default. + * @param dflt Default value. * @return Value. */ private static String systemOrEnv(String name, String dflt) { @@ -158,7 +210,7 @@ public class HadoopClasspathUtils { if (res == null) res = System.getenv(name); - return res == null ? dflt : res; + return res != null ? res : dflt; } /** @@ -167,7 +219,7 @@ public class HadoopClasspathUtils { * @param path The directory path. * @return {@code True} if the given path denotes an existing directory. */ - private static boolean directoryExists(String path) { + public static boolean exists(String path) { if (path == null) return false; @@ -177,6 +229,16 @@ public class HadoopClasspathUtils { } /** + * Check if string is empty. + * + * @param val Value. + * @return {@code True} if empty. + */ + private static boolean isEmpty(String val) { + return val == null || val.isEmpty(); + } + + /** * Simple pair-like structure to hold directory name and a mask assigned to it. */ private static class SearchDirectory { @@ -196,7 +258,7 @@ public class HadoopClasspathUtils { this.dir = dir; this.filter = filter; - if (!directoryExists(dir.getAbsolutePath())) + if (!exists(dir.getAbsolutePath())) throw new IOException("Directory cannot be read: " + dir.getAbsolutePath()); } @@ -230,4 +292,4 @@ public class HadoopClasspathUtils { return filter != null; } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/34798d3a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java index b3e2d12..a90007f 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java @@ -25,27 +25,42 @@ public class HadoopLocations { private final String home; /** Common home. */ - private final String commonHome; + private final String common; /** HDFS home. */ - private final String hdfsHome; + private final String hdfs; /** Mapred home. */ - private final String mapredHome; + private final String mapred; + + /** Whether common home exists. */ + private final boolean commonExists; + + /** Whether HDFS home exists. */ + private final boolean hdfsExists; + + /** Whether mapred home exists. */ + private final boolean mapredExists; /** * Constructor. * * @param home Hadoop home. - * @param commonHome Common home. - * @param hdfsHome HDFS home. - * @param mapredHome Mapred home. + * @param common Common home. + * @param hdfs HDFS home. + * @param mapred Mapred home. */ - public HadoopLocations(String home, String commonHome, String hdfsHome, String mapredHome) { + public HadoopLocations(String home, String common, String hdfs, String mapred) { + assert common != null && hdfs != null && mapred != null; + this.home = home; - this.commonHome = commonHome; - this.hdfsHome = hdfsHome; - this.mapredHome = mapredHome; + this.common = common; + this.hdfs = hdfs; + this.mapred = mapred; + + commonExists = HadoopClasspathUtils.exists(common); + hdfsExists = HadoopClasspathUtils.exists(hdfs); + mapredExists = HadoopClasspathUtils.exists(mapred); } /** @@ -58,21 +73,51 @@ public class HadoopLocations { /** * @return Common home. */ - public String commonHome() { - return commonHome; + public String common() { + return common; } /** * @return HDFS home. */ - public String hdfsHome() { - return hdfsHome; + public String hdfs() { + return hdfs; } /** * @return Mapred home. */ - public String mapredHome() { - return mapredHome; + public String mapred() { + return mapred; + } + + /** + * @return Whether common home exists. + */ + public boolean commonExists() { + return commonExists; + } + + /** + * @return Whether HDFS home exists. + */ + public boolean hdfsExists() { + return hdfsExists; + } + + /** + * @return Whether mapred home exists. + */ + public boolean mapredExists() { + return mapredExists; + } + + /** + * Whether all required directories exists. + * + * @return {@code True} if exists. + */ + public boolean valid() { + return commonExists && hdfsExists && mapredExists; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/34798d3a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java index bb10565..b9c20c3 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker; import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffle; import org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopEmbeddedTaskExecutor; import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -190,14 +191,13 @@ public class HadoopProcessor extends HadoopProcessorAdapter { @Override public void validateEnvironment() throws IgniteCheckedException { // Perform some static checks as early as possible, so that any recoverable exceptions are thrown here. try { - HadoopLocations loc = HadoopClasspathUtils.hadoopLocations(); + HadoopLocations loc = HadoopClasspathUtils.locations(); - if (loc.home() != null) - U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home()); + if (!F.isEmpty(loc.home())) + U.quietAndInfo(log, HadoopClasspathUtils.HOME + " is set to " + loc.home()); - U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome()); - U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome()); - U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome()); + U.quietAndInfo(log, "Resolved Hadoop classpath locations: " + loc.common() + ", " + loc.hdfs() + ", " + + loc.mapred()); } catch (IOException ioe) { throw new IgniteCheckedException(ioe.getMessage(), ioe);
