[14/50] [abbrv] oozie git commit: OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini)
OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cdce5a1 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cdce5a1 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cdce5a1 Branch: refs/heads/oya Commit: 0cdce5a1645799d01520176c17da6ab7b9739c73 Parents: f861071 Author: Rohini Palaniswamy Authored: Mon Sep 19 10:07:57 2016 -0700 Committer: Rohini Palaniswamy Committed: Mon Sep 19 10:07:57 2016 -0700 -- .../oozie/action/hadoop/SparkActionExecutor.java | 6 ++ core/src/main/resources/oozie-default.xml | 11 +++ release-log.txt | 1 + .../org/apache/oozie/action/hadoop/LauncherMain.java | 14 ++ .../org/apache/oozie/action/hadoop/ShellMain.java | 13 ++--- .../org/apache/oozie/action/hadoop/SparkMain.java | 13 + 6 files changed, 47 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 97355fd..15a641b 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.Services; import org.apache.oozie.service.SparkConfigurationService; @@ -49,6 +50,7 @@ public class SparkActionExecutor extends JavaActionExecutor { public static final String SPARK_CLASS = "oozie.spark.class"; public static final String SPARK_JAR = "oozie.spark.jar"; public static final String MAPRED_CHILD_ENV = "mapred.child.env"; +private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir"; public SparkActionExecutor() { super("spark"); @@ -95,6 +97,10 @@ public class SparkActionExecutor extends JavaActionExecutor { actionConf.set(SPARK_OPTS, sparkOptsSb.toString().trim()); } +// Setting if SparkMain should setup hadoop config *-site.xml +boolean setupHadoopConf = actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, + ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR)); +actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, setupHadoopConf); return actionConf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/resources/oozie-default.xml -- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 40a5fe5..e71ebe3 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1815,6 +1815,17 @@ will be the requeue interval for the actions which are waiting for a long time w +oozie.action.spark.setup.hadoop.conf.dir +false + +Oozie action.xml (oozie.action.conf.xml) contains all the hadoop configuration and user provided configurations. +This property will allow users to copy Oozie action.xml as hadoop *-site configurations files. The advantage is, +user need not to manage these files into spark sharelib. If user wants to manage the hadoop configurations +themselves, it should should disable it. + + + + oozie.action.shell.setup.hadoop.conf.dir false http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 681e5ee..7851e81 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini) OOZIE-2588 Support getting credentials for same cluster hcat when credentials config is empty (satishsaley via rohini) OOZIE-253
oozie git commit: OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini)
Repository: oozie Updated Branches: refs/heads/master f86107155 -> 0cdce5a16 OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cdce5a1 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cdce5a1 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cdce5a1 Branch: refs/heads/master Commit: 0cdce5a1645799d01520176c17da6ab7b9739c73 Parents: f861071 Author: Rohini Palaniswamy Authored: Mon Sep 19 10:07:57 2016 -0700 Committer: Rohini Palaniswamy Committed: Mon Sep 19 10:07:57 2016 -0700 -- .../oozie/action/hadoop/SparkActionExecutor.java | 6 ++ core/src/main/resources/oozie-default.xml | 11 +++ release-log.txt | 1 + .../org/apache/oozie/action/hadoop/LauncherMain.java | 14 ++ .../org/apache/oozie/action/hadoop/ShellMain.java | 13 ++--- .../org/apache/oozie/action/hadoop/SparkMain.java | 13 + 6 files changed, 47 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 97355fd..15a641b 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.Services; import org.apache.oozie.service.SparkConfigurationService; @@ -49,6 +50,7 @@ public class SparkActionExecutor extends JavaActionExecutor { public static final String SPARK_CLASS = "oozie.spark.class"; public static final String SPARK_JAR = "oozie.spark.jar"; public static final String MAPRED_CHILD_ENV = "mapred.child.env"; +private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir"; public SparkActionExecutor() { super("spark"); @@ -95,6 +97,10 @@ public class SparkActionExecutor extends JavaActionExecutor { actionConf.set(SPARK_OPTS, sparkOptsSb.toString().trim()); } +// Setting if SparkMain should setup hadoop config *-site.xml +boolean setupHadoopConf = actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, + ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR)); +actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, setupHadoopConf); return actionConf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/resources/oozie-default.xml -- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 40a5fe5..e71ebe3 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1815,6 +1815,17 @@ will be the requeue interval for the actions which are waiting for a long time w +oozie.action.spark.setup.hadoop.conf.dir +false + +Oozie action.xml (oozie.action.conf.xml) contains all the hadoop configuration and user provided configurations. +This property will allow users to copy Oozie action.xml as hadoop *-site configurations files. The advantage is, +user need not to manage these files into spark sharelib. If user wants to manage the hadoop configurations +themselves, it should should disable it. + + + + oozie.action.shell.setup.hadoop.conf.dir false http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 681e5ee..7851e81 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini) OOZIE-2588 Support getting credentials for sam