IGNITE-3383: Hadoop: Improved Hadoop component start logic.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/00ce2097 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/00ce2097 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/00ce2097 Branch: refs/heads/ignite-1232 Commit: 00ce2097f57b7f3def04da9449c68b17135cbeaa Parents: 26eec3a Author: vozerov-gridgain <[email protected]> Authored: Tue Jun 28 16:33:09 2016 +0300 Committer: vozerov-gridgain <[email protected]> Committed: Thu Jun 30 10:19:10 2016 +0300 ---------------------------------------------------------------------- .../apache/ignite/internal/IgniteKernal.java | 48 ++++++++++++------ .../processors/hadoop/HadoopNoopProcessor.java | 5 ++ .../hadoop/HadoopProcessorAdapter.java | 7 +++ .../processors/hadoop/HadoopClassLoader.java | 2 +- .../processors/hadoop/HadoopProcessor.java | 52 +++++++++----------- 5 files changed, 68 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/00ce2097/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index e77875e..0c1bd93 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -115,7 +115,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor; import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor; import org.apache.ignite.internal.processors.hadoop.Hadoop; -import org.apache.ignite.internal.processors.hadoop.HadoopNoopProcessor; +import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter; import org.apache.ignite.internal.processors.job.GridJobProcessor; import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor; import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; @@ -1111,26 +1111,44 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { * Create Hadoop component. * * @return Non-null Hadoop component: workable or no-op. - * @throws IgniteCheckedException if the component is mandatory and cannot be initialized. + * @throws IgniteCheckedException If the component is mandatory and cannot be initialized. */ - private GridComponent createHadoopComponent() throws IgniteCheckedException { - GridComponent cmp; + private HadoopProcessorAdapter createHadoopComponent() throws IgniteCheckedException { + boolean mandatory = cfg.getHadoopConfiguration() != null; - if (cfg.isPeerClassLoadingEnabled()) { - cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false); + if (mandatory) { + if (cfg.isPeerClassLoadingEnabled()) + throw new IgniteCheckedException("Hadoop module cannot be used with peer class loading enabled " + + "(set IgniteConfiguration.peerClassLoadingEnabled to \"false\")."); - if (!(cmp instanceof HadoopNoopProcessor)) { - U.warn(log, "Hadoop module is found in classpath, but it will not be started because peer class " + - "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled to \"false\" to start " + - "Hadoop module)."); + return IgniteComponentType.HADOOP.createIfInClassPath(ctx, true); + } + else { + HadoopProcessorAdapter cmp = null; - cmp = IgniteComponentType.HADOOP.create(ctx, true/*no-op*/); + if (IgniteComponentType.HADOOP.inClassPath() && cfg.isPeerClassLoadingEnabled()) { + U.warn(log, "Hadoop module is found in classpath, but will not be started because peer class " + + "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled to \"false\" if you want " + + "to use Hadoop module)."); } - } - else - cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, cfg.getHadoopConfiguration() != null); + else { + cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false); + + try { + cmp.validateEnvironment(); + } + catch (IgniteException | IgniteCheckedException e) { + U.quietAndWarn(log, "Hadoop module will not start due to exception: " + e.getMessage()); + + cmp = null; + } + } + + if (cmp == null) + cmp = IgniteComponentType.HADOOP.create(ctx, true); - return cmp; + return cmp; + } } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/00ce2097/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java index e772787..501870a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java @@ -82,4 +82,9 @@ public class HadoopNoopProcessor extends HadoopProcessorAdapter { return new IllegalStateException("Hadoop module is not loaded (please ensure that ignite-hadoop.jar is in " + "classpath and IgniteConfiguration.peerClassLoadingEnabled is set to false)."); } + + /** {@inheritDoc} */ + @Override public void validateEnvironment() throws IgniteCheckedException { + // No-oop. + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/00ce2097/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java index 3d381c3..f914639 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java @@ -94,4 +94,11 @@ public abstract class HadoopProcessorAdapter extends GridProcessorAdapter { * @throws IgniteCheckedException If failed. */ public abstract boolean kill(HadoopJobId jobId) throws IgniteCheckedException; + + /** + * Validate Hadoop environment. + * + * @throws IgniteCheckedException If failed. + */ + public abstract void validateEnvironment() throws IgniteCheckedException; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/00ce2097/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java index 1c844c4..389de8c 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java @@ -501,7 +501,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { hadoopUrls = HadoopClasspathUtils.classpathUrls(); } catch (IOException e) { - throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations.", e); + throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations: " + e.getMessage(), e); } hadoopJars = hadoopUrls; http://git-wip-us.apache.org/repos/asf/ignite/blob/00ce2097/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java index c45953e..7fc7499 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java @@ -50,6 +50,8 @@ public class HadoopProcessor extends HadoopProcessorAdapter { private Hadoop hadoop; /** + * Constructor. + * * @param ctx Kernal context. */ public HadoopProcessor(GridKernalContext ctx) { @@ -70,24 +72,6 @@ public class HadoopProcessor extends HadoopProcessorAdapter { initializeDefaults(cfg); - validate(cfg); - - try { - HadoopLocations loc = HadoopClasspathUtils.hadoopLocations(); - - if (loc.home() != null) - U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home()); - - U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome()); - U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome()); - U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome()); - } - catch (IOException ioe) { - throw new IgniteCheckedException(ioe); - } - - HadoopClassLoader.hadoopUrls(); - hctx = new HadoopContext( ctx, cfg, @@ -204,6 +188,26 @@ public class HadoopProcessor extends HadoopProcessorAdapter { return hctx.jobTracker().killJob(jobId); } + /** {@inheritDoc} */ + @Override public void validateEnvironment() throws IgniteCheckedException { + // Perform some static checks as early as possible, so that any recoverable exceptions are thrown here. + try { + HadoopLocations loc = HadoopClasspathUtils.hadoopLocations(); + + if (loc.home() != null) + U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home()); + + U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome()); + U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome()); + U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome()); + } + catch (IOException ioe) { + throw new IgniteCheckedException(ioe.getMessage(), ioe); + } + + HadoopClassLoader.hadoopUrls(); + } + /** * Initializes default hadoop configuration. * @@ -213,16 +217,4 @@ public class HadoopProcessor extends HadoopProcessorAdapter { if (cfg.getMapReducePlanner() == null) cfg.setMapReducePlanner(new IgniteHadoopMapReducePlanner()); } - - /** - * Validates Grid and Hadoop configuration for correctness. - * - * @param hadoopCfg Hadoop configuration. - * @throws IgniteCheckedException If failed. - */ - private void validate(HadoopConfiguration hadoopCfg) throws IgniteCheckedException { - if (ctx.config().isPeerClassLoadingEnabled()) - throw new IgniteCheckedException("Peer class loading cannot be used with Hadoop (disable it using " + - "IgniteConfiguration.setPeerClassLoadingEnabled())."); - } } \ No newline at end of file
