Repository: oozie
Updated Branches:
  refs/heads/master f86107155 -> 0cdce5a16


OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark 
launcher (abhishekbafna via rohini)


Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cdce5a1
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cdce5a1
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cdce5a1

Branch: refs/heads/master
Commit: 0cdce5a1645799d01520176c17da6ab7b9739c73
Parents: f861071
Author: Rohini Palaniswamy <roh...@apache.org>
Authored: Mon Sep 19 10:07:57 2016 -0700
Committer: Rohini Palaniswamy <roh...@apache.org>
Committed: Mon Sep 19 10:07:57 2016 -0700

----------------------------------------------------------------------
 .../oozie/action/hadoop/SparkActionExecutor.java      |  6 ++++++
 core/src/main/resources/oozie-default.xml             | 11 +++++++++++
 release-log.txt                                       |  1 +
 .../org/apache/oozie/action/hadoop/LauncherMain.java  | 14 ++++++++++++++
 .../org/apache/oozie/action/hadoop/ShellMain.java     | 13 ++-----------
 .../org/apache/oozie/action/hadoop/SparkMain.java     | 13 +++++++++++++
 6 files changed, 47 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java 
b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 97355fd..15a641b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.oozie.action.ActionExecutorException;
 import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.ConfigurationService;
 import org.apache.oozie.service.HadoopAccessorException;
 import org.apache.oozie.service.Services;
 import org.apache.oozie.service.SparkConfigurationService;
@@ -49,6 +50,7 @@ public class SparkActionExecutor extends JavaActionExecutor {
     public static final String SPARK_CLASS = "oozie.spark.class";
     public static final String SPARK_JAR = "oozie.spark.jar";
     public static final String MAPRED_CHILD_ENV = "mapred.child.env";
+    private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = 
"oozie.action.spark.setup.hadoop.conf.dir";
 
     public SparkActionExecutor() {
         super("spark");
@@ -95,6 +97,10 @@ public class SparkActionExecutor extends JavaActionExecutor {
             actionConf.set(SPARK_OPTS, sparkOptsSb.toString().trim());
         }
 
+        // Setting if SparkMain should setup hadoop config *-site.xml
+        boolean setupHadoopConf = 
actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR,
+                
ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR));
+        actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, 
setupHadoopConf);
         return actionConf;
     }
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml 
b/core/src/main/resources/oozie-default.xml
index 40a5fe5..e71ebe3 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1815,6 +1815,17 @@ will be the requeue interval for the actions which are 
waiting for a long time w
     </property>
 
     <property>
+        <name>oozie.action.spark.setup.hadoop.conf.dir</name>
+        <value>false</value>
+        <description>
+            Oozie action.xml (oozie.action.conf.xml) contains all the hadoop 
configuration and user provided configurations.
+            This property will allow users to copy Oozie action.xml as hadoop 
*-site configurations files. The advantage is,
+            user need not to manage these files into spark sharelib. If user 
wants to manage the hadoop configurations
+            themselves, it should should disable it.
+        </description>
+    </property>
+
+    <property>
         <name>oozie.action.shell.setup.hadoop.conf.dir</name>
         <value>false</value>
         <description>

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 681e5ee..7851e81 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 4.3.0 release (trunk - unreleased)
 
+OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark 
launcher (abhishekbafna via rohini)
 OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini)
 OOZIE-2588 Support getting credentials for same cluster hcat when credentials 
config is empty (satishsaley via rohini)
 OOZIE-2538 Update HttpClient versions to close security vulnerabilities 
(abhishekbafna via rkanter)

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --git 
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java 
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index fb190d6..054f8ea 100644
--- 
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ 
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.StringWriter;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Map;
 import java.util.Properties;
@@ -43,6 +44,8 @@ public abstract class LauncherMain {
 
     public static final String HADOOP_JOBS = "hadoopJobs";
     public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags";
+    protected static String[] HADOOP_SITE_FILES = new String[]
+            {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", 
"yarn-site.xml"};
 
     protected static void run(Class<? extends LauncherMain> klass, String[] 
args) throws Exception {
         LauncherMain main = klass.newInstance();
@@ -247,6 +250,17 @@ public abstract class LauncherMain {
             }
         }
     }
+
+    protected void writeHadoopConfig(String actionXml, File basrDir) throws 
IOException {
+        File actionXmlFile = new File(actionXml);
+        System.out.println("Copying " + actionXml + " to " + basrDir + "/" + 
Arrays.toString(HADOOP_SITE_FILES));
+        basrDir.mkdirs();
+        File[] dstFiles = new File[HADOOP_SITE_FILES.length];
+        for (int i = 0; i < dstFiles.length; i++) {
+            dstFiles[i] = new File(basrDir, HADOOP_SITE_FILES[i]);
+        }
+        copyFileMultiplex(actionXmlFile, dstFiles);
+    }
 }
 
 class LauncherMainException extends Exception {

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
----------------------------------------------------------------------
diff --git 
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java 
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
index c4a6e9b..5e80d00 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
@@ -29,7 +29,6 @@ import java.io.StringReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -50,7 +49,6 @@ public class ShellMain extends LauncherMain {
     private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR";
     private static final String YARN_CONF_DIR = "YARN_CONF_DIR";
 
-    private static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", 
"hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"};
     private static String LOG4J_PROPERTIES = "log4j.properties";
 
     /**
@@ -88,7 +86,7 @@ public class ShellMain extends LauncherMain {
         ProcessBuilder builder = new ProcessBuilder(cmdArray);
         Map<String, String> envp = getEnvMap(builder.environment(), 
actionConf);
 
-        // Getting the Ccurrent working dir and setting it to processbuilder
+        // Getting the Current working dir and setting it to processbuilder
         File currDir = new File("dummy").getAbsoluteFile().getParentFile();
         System.out.println("Current working dir " + currDir);
         builder.directory(currDir);
@@ -140,15 +138,8 @@ public class ShellMain extends LauncherMain {
         if (actionConf.getBoolean(CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR, 
false)) {
             String actionXml = envp.get(OOZIE_ACTION_CONF_XML);
             if (actionXml != null) {
-                File actionXmlFile = new File(actionXml);
                 File confDir = new File(currDir, "oozie-hadoop-conf-" + 
System.currentTimeMillis());
-                System.out.println("Copying " + actionXml + " to " + confDir + 
"/" + Arrays.toString(HADOOP_SITE_FILES));
-                confDir.mkdirs();
-                File[] dstFiles = new File[HADOOP_SITE_FILES.length];
-                for (int i = 0; i < dstFiles.length; i++) {
-                    dstFiles[i] = new File(confDir, HADOOP_SITE_FILES[i]);
-                }
-                copyFileMultiplex(actionXmlFile, dstFiles);
+                writeHadoopConfig(actionXml, confDir);
                 if 
(actionConf.getBoolean(CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR_WRITE_LOG4J_PROPERTIES,
 true)) {
                     System.out.println("Writing " + LOG4J_PROPERTIES + " to " 
+ confDir);
                     writeLoggerProperties(actionConf, confDir);

http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git 
a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java 
b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 3acaef9..38e8e8c 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -48,6 +48,7 @@ public class SparkMain extends LauncherMain {
     private static final String DRIVER_CLASSPATH = 
"spark.driver.extraClassPath=";
     private static final String HIVE_SECURITY_TOKEN = 
"spark.yarn.security.tokens.hive.enabled";
     private static final String HBASE_SECURITY_TOKEN = 
"spark.yarn.security.tokens.hbase.enabled";
+    private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = 
"oozie.action.spark.setup.hadoop.conf.dir";
     private static final String PWD = "$PWD" + File.separator + "*";
     private static final Pattern[] PYSPARK_DEP_FILE_PATTERN = { 
Pattern.compile("py4\\S*src.zip"),
             Pattern.compile("pyspark.zip") };
@@ -63,6 +64,7 @@ public class SparkMain extends LauncherMain {
     protected void run(String[] args) throws Exception {
         boolean isPyspark = false;
         Configuration actionConf = loadActionConf();
+        prepareHadoopConfig(actionConf);
 
         setYarnTag(actionConf);
         LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
@@ -214,6 +216,17 @@ public class SparkMain extends LauncherMain {
         }
     }
 
+    private void prepareHadoopConfig(Configuration actionConf) throws 
IOException {
+        // Copying oozie.action.conf.xml into hadoop configuration *-site 
files.
+        if (actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, 
false)) {
+            String actionXml = System.getProperty("oozie.action.conf.xml");
+            if (actionXml != null) {
+                File currentDir = new File(actionXml).getParentFile();
+                writeHadoopConfig(actionXml, currentDir);
+            }
+        }
+    }
+
     /**
      * SparkActionExecutor sets the SPARK_HOME environment variable to the 
local directory.
      * Spark is looking for the pyspark.zip and py4j-VERSION-src.zip files in 
the python/lib folder under SPARK_HOME.

Reply via email to