Repository: oozie Updated Branches: refs/heads/master f86107155 -> 0cdce5a16
OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cdce5a1 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cdce5a1 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cdce5a1 Branch: refs/heads/master Commit: 0cdce5a1645799d01520176c17da6ab7b9739c73 Parents: f861071 Author: Rohini Palaniswamy <[email protected]> Authored: Mon Sep 19 10:07:57 2016 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Mon Sep 19 10:07:57 2016 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/SparkActionExecutor.java | 6 ++++++ core/src/main/resources/oozie-default.xml | 11 +++++++++++ release-log.txt | 1 + .../org/apache/oozie/action/hadoop/LauncherMain.java | 14 ++++++++++++++ .../org/apache/oozie/action/hadoop/ShellMain.java | 13 ++----------- .../org/apache/oozie/action/hadoop/SparkMain.java | 13 +++++++++++++ 6 files changed, 47 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 97355fd..15a641b 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.Services; import org.apache.oozie.service.SparkConfigurationService; @@ -49,6 +50,7 @@ public class SparkActionExecutor extends JavaActionExecutor { public static final String SPARK_CLASS = "oozie.spark.class"; public static final String SPARK_JAR = "oozie.spark.jar"; public static final String MAPRED_CHILD_ENV = "mapred.child.env"; + private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir"; public SparkActionExecutor() { super("spark"); @@ -95,6 +97,10 @@ public class SparkActionExecutor extends JavaActionExecutor { actionConf.set(SPARK_OPTS, sparkOptsSb.toString().trim()); } + // Setting if SparkMain should setup hadoop config *-site.xml + boolean setupHadoopConf = actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, + ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR)); + actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, setupHadoopConf); return actionConf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 40a5fe5..e71ebe3 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1815,6 +1815,17 @@ will be the requeue interval for the actions which are waiting for a long time w </property> <property> + <name>oozie.action.spark.setup.hadoop.conf.dir</name> + <value>false</value> + <description> + Oozie action.xml (oozie.action.conf.xml) contains all the hadoop configuration and user provided configurations. + This property will allow users to copy Oozie action.xml as hadoop *-site configurations files. The advantage is, + user need not to manage these files into spark sharelib. If user wants to manage the hadoop configurations + themselves, it should should disable it. + </description> + </property> + + <property> <name>oozie.action.shell.setup.hadoop.conf.dir</name> <value>false</value> <description> http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 681e5ee..7851e81 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini) OOZIE-2588 Support getting credentials for same cluster hcat when credentials config is empty (satishsaley via rohini) OOZIE-2538 Update HttpClient versions to close security vulnerabilities (abhishekbafna via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index fb190d6..054f8ea 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; +import java.util.Arrays; import java.util.Collection; import java.util.Map; import java.util.Properties; @@ -43,6 +44,8 @@ public abstract class LauncherMain { public static final String HADOOP_JOBS = "hadoopJobs"; public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags"; + protected static String[] HADOOP_SITE_FILES = new String[] + {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"}; protected static void run(Class<? extends LauncherMain> klass, String[] args) throws Exception { LauncherMain main = klass.newInstance(); @@ -247,6 +250,17 @@ public abstract class LauncherMain { } } } + + protected void writeHadoopConfig(String actionXml, File basrDir) throws IOException { + File actionXmlFile = new File(actionXml); + System.out.println("Copying " + actionXml + " to " + basrDir + "/" + Arrays.toString(HADOOP_SITE_FILES)); + basrDir.mkdirs(); + File[] dstFiles = new File[HADOOP_SITE_FILES.length]; + for (int i = 0; i < dstFiles.length; i++) { + dstFiles[i] = new File(basrDir, HADOOP_SITE_FILES[i]); + } + copyFileMultiplex(actionXmlFile, dstFiles); + } } class LauncherMainException extends Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java index c4a6e9b..5e80d00 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java @@ -29,7 +29,6 @@ import java.io.StringReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -50,7 +49,6 @@ public class ShellMain extends LauncherMain { private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR"; private static final String YARN_CONF_DIR = "YARN_CONF_DIR"; - private static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"}; private static String LOG4J_PROPERTIES = "log4j.properties"; /** @@ -88,7 +86,7 @@ public class ShellMain extends LauncherMain { ProcessBuilder builder = new ProcessBuilder(cmdArray); Map<String, String> envp = getEnvMap(builder.environment(), actionConf); - // Getting the Ccurrent working dir and setting it to processbuilder + // Getting the Current working dir and setting it to processbuilder File currDir = new File("dummy").getAbsoluteFile().getParentFile(); System.out.println("Current working dir " + currDir); builder.directory(currDir); @@ -140,15 +138,8 @@ public class ShellMain extends LauncherMain { if (actionConf.getBoolean(CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR, false)) { String actionXml = envp.get(OOZIE_ACTION_CONF_XML); if (actionXml != null) { - File actionXmlFile = new File(actionXml); File confDir = new File(currDir, "oozie-hadoop-conf-" + System.currentTimeMillis()); - System.out.println("Copying " + actionXml + " to " + confDir + "/" + Arrays.toString(HADOOP_SITE_FILES)); - confDir.mkdirs(); - File[] dstFiles = new File[HADOOP_SITE_FILES.length]; - for (int i = 0; i < dstFiles.length; i++) { - dstFiles[i] = new File(confDir, HADOOP_SITE_FILES[i]); - } - copyFileMultiplex(actionXmlFile, dstFiles); + writeHadoopConfig(actionXml, confDir); if (actionConf.getBoolean(CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR_WRITE_LOG4J_PROPERTIES, true)) { System.out.println("Writing " + LOG4J_PROPERTIES + " to " + confDir); writeLoggerProperties(actionConf, confDir); http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 3acaef9..38e8e8c 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -48,6 +48,7 @@ public class SparkMain extends LauncherMain { private static final String DRIVER_CLASSPATH = "spark.driver.extraClassPath="; private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled"; private static final String HBASE_SECURITY_TOKEN = "spark.yarn.security.tokens.hbase.enabled"; + private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir"; private static final String PWD = "$PWD" + File.separator + "*"; private static final Pattern[] PYSPARK_DEP_FILE_PATTERN = { Pattern.compile("py4\\S*src.zip"), Pattern.compile("pyspark.zip") }; @@ -63,6 +64,7 @@ public class SparkMain extends LauncherMain { protected void run(String[] args) throws Exception { boolean isPyspark = false; Configuration actionConf = loadActionConf(); + prepareHadoopConfig(actionConf); setYarnTag(actionConf); LauncherMainHadoopUtils.killChildYarnJobs(actionConf); @@ -214,6 +216,17 @@ public class SparkMain extends LauncherMain { } } + private void prepareHadoopConfig(Configuration actionConf) throws IOException { + // Copying oozie.action.conf.xml into hadoop configuration *-site files. + if (actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, false)) { + String actionXml = System.getProperty("oozie.action.conf.xml"); + if (actionXml != null) { + File currentDir = new File(actionXml).getParentFile(); + writeHadoopConfig(actionXml, currentDir); + } + } + } + /** * SparkActionExecutor sets the SPARK_HOME environment variable to the local directory. * Spark is looking for the pyspark.zip and py4j-VERSION-src.zip files in the python/lib folder under SPARK_HOME.
