[14/50] [abbrv] oozie git commit: OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini)

2016-10-10 Thread gezapeti 
OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark 
launcher (abhishekbafna via rohini)


Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cdce5a1
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cdce5a1
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cdce5a1

Branch: refs/heads/oya
Commit: 0cdce5a1645799d01520176c17da6ab7b9739c73
Parents: f861071
Author: Rohini Palaniswamy 
Authored: Mon Sep 19 10:07:57 2016 -0700
Committer: Rohini Palaniswamy 
Committed: Mon Sep 19 10:07:57 2016 -0700

--
 .../oozie/action/hadoop/SparkActionExecutor.java  |  6 ++
 core/src/main/resources/oozie-default.xml | 11 +++
 release-log.txt   |  1 +
 .../org/apache/oozie/action/hadoop/LauncherMain.java  | 14 ++
 .../org/apache/oozie/action/hadoop/ShellMain.java | 13 ++---
 .../org/apache/oozie/action/hadoop/SparkMain.java | 13 +
 6 files changed, 47 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
--
diff --git 
a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java 
b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 97355fd..15a641b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.oozie.action.ActionExecutorException;
 import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.ConfigurationService;
 import org.apache.oozie.service.HadoopAccessorException;
 import org.apache.oozie.service.Services;
 import org.apache.oozie.service.SparkConfigurationService;
@@ -49,6 +50,7 @@ public class SparkActionExecutor extends JavaActionExecutor {
 public static final String SPARK_CLASS = "oozie.spark.class";
 public static final String SPARK_JAR = "oozie.spark.jar";
 public static final String MAPRED_CHILD_ENV = "mapred.child.env";
+private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = 
"oozie.action.spark.setup.hadoop.conf.dir";
 
 public SparkActionExecutor() {
 super("spark");
@@ -95,6 +97,10 @@ public class SparkActionExecutor extends JavaActionExecutor {
 actionConf.set(SPARK_OPTS, sparkOptsSb.toString().trim());
 }
 
+// Setting if SparkMain should setup hadoop config *-site.xml
+boolean setupHadoopConf = 
actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR,
+
ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR));
+actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, 
setupHadoopConf);
 return actionConf;
 }
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/resources/oozie-default.xml
--
diff --git a/core/src/main/resources/oozie-default.xml 
b/core/src/main/resources/oozie-default.xml
index 40a5fe5..e71ebe3 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1815,6 +1815,17 @@ will be the requeue interval for the actions which are 
waiting for a long time w
 
 
 
+oozie.action.spark.setup.hadoop.conf.dir
+false
+
+Oozie action.xml (oozie.action.conf.xml) contains all the hadoop 
configuration and user provided configurations.
+This property will allow users to copy Oozie action.xml as hadoop 
*-site configurations files. The advantage is,
+user need not to manage these files into spark sharelib. If user 
wants to manage the hadoop configurations
+themselves, it should should disable it.
+
+
+
+
 oozie.action.shell.setup.hadoop.conf.dir
 false
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/release-log.txt
--
diff --git a/release-log.txt b/release-log.txt
index 681e5ee..7851e81 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 4.3.0 release (trunk - unreleased)
 
+OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark 
launcher (abhishekbafna via rohini)
 OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini)
 OOZIE-2588 Support getting credentials for same cluster hcat when credentials 
config is

oozie git commit: OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini)

2016-09-19 Thread rohini 
Repository: oozie
Updated Branches:
  refs/heads/master f86107155 -> 0cdce5a16


OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark 
launcher (abhishekbafna via rohini)


Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cdce5a1
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cdce5a1
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cdce5a1

Branch: refs/heads/master
Commit: 0cdce5a1645799d01520176c17da6ab7b9739c73
Parents: f861071
Author: Rohini Palaniswamy 
Authored: Mon Sep 19 10:07:57 2016 -0700
Committer: Rohini Palaniswamy 
Committed: Mon Sep 19 10:07:57 2016 -0700

--
 .../oozie/action/hadoop/SparkActionExecutor.java  |  6 ++
 core/src/main/resources/oozie-default.xml | 11 +++
 release-log.txt   |  1 +
 .../org/apache/oozie/action/hadoop/LauncherMain.java  | 14 ++
 .../org/apache/oozie/action/hadoop/ShellMain.java | 13 ++---
 .../org/apache/oozie/action/hadoop/SparkMain.java | 13 +
 6 files changed, 47 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
--
diff --git 
a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java 
b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 97355fd..15a641b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.oozie.action.ActionExecutorException;
 import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.ConfigurationService;
 import org.apache.oozie.service.HadoopAccessorException;
 import org.apache.oozie.service.Services;
 import org.apache.oozie.service.SparkConfigurationService;
@@ -49,6 +50,7 @@ public class SparkActionExecutor extends JavaActionExecutor {
 public static final String SPARK_CLASS = "oozie.spark.class";
 public static final String SPARK_JAR = "oozie.spark.jar";
 public static final String MAPRED_CHILD_ENV = "mapred.child.env";
+private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = 
"oozie.action.spark.setup.hadoop.conf.dir";
 
 public SparkActionExecutor() {
 super("spark");
@@ -95,6 +97,10 @@ public class SparkActionExecutor extends JavaActionExecutor {
 actionConf.set(SPARK_OPTS, sparkOptsSb.toString().trim());
 }
 
+// Setting if SparkMain should setup hadoop config *-site.xml
+boolean setupHadoopConf = 
actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR,
+
ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR));
+actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, 
setupHadoopConf);
 return actionConf;
 }
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/resources/oozie-default.xml
--
diff --git a/core/src/main/resources/oozie-default.xml 
b/core/src/main/resources/oozie-default.xml
index 40a5fe5..e71ebe3 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1815,6 +1815,17 @@ will be the requeue interval for the actions which are 
waiting for a long time w
 
 
 
+oozie.action.spark.setup.hadoop.conf.dir
+false
+
+Oozie action.xml (oozie.action.conf.xml) contains all the hadoop 
configuration and user provided configurations.
+This property will allow users to copy Oozie action.xml as hadoop 
*-site configurations files. The advantage is,
+user need not to manage these files into spark sharelib. If user 
wants to manage the hadoop configurations
+themselves, it should should disable it.
+
+
+
+
 oozie.action.shell.setup.hadoop.conf.dir
 false
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/release-log.txt
--
diff --git a/release-log.txt b/release-log.txt
index 681e5ee..7851e81 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 4.3.0 release (trunk - unreleased)
 
+OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark 
launcher (abhishekbafna via rohini)
 OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini)