Repository: oozie Updated Branches: refs/heads/master d330d4066 -> 376cdb3d8
OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/376cdb3d Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/376cdb3d Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/376cdb3d Branch: refs/heads/master Commit: 376cdb3d8cd4f3a5c2c3f3e771df19c50e57b8f4 Parents: d330d40 Author: Rohini Palaniswamy <[email protected]> Authored: Tue Sep 27 16:22:39 2016 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Tue Sep 27 16:22:39 2016 -0700 ---------------------------------------------------------------------- release-log.txt | 1 + .../src/main/java/org/apache/oozie/action/hadoop/JavaMain.java | 1 + .../main/java/org/apache/oozie/action/hadoop/LauncherMain.java | 2 +- .../src/main/java/org/apache/oozie/action/hadoop/ShellMain.java | 1 + .../src/main/java/org/apache/oozie/action/hadoop/SparkMain.java | 5 +++++ 5 files changed, 9 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b03a61a..1ca7e2e 100644 --- a/release-log.txt +++ b/release-log.txt @@ -3,6 +3,7 @@ -- Oozie 4.3.0 release +OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini) OOZIE-2501 ZK reentrant lock doesn't work for few cases (puru) OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini) OOZIE-2678 Oozie job -kill doesn't work with tez jobs (abhishekbafna via rohini) http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java index 16bd4e6..30d68e2 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java @@ -42,6 +42,7 @@ public class JavaMain extends LauncherMain { setYarnTag(actionConf); setApplicationTags(actionConf, TEZ_APPLICATION_TAGS); + setApplicationTags(actionConf, SPARK_YARN_TAGS); LauncherMainHadoopUtils.killChildYarnJobs(actionConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 815f60b..785ca5e 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -31,7 +31,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashSet; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -52,6 +51,7 @@ public abstract class LauncherMain { public static final String HADOOP_JOBS = "hadoopJobs"; public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags"; public static final String TEZ_APPLICATION_TAGS = "tez.application.tags"; + public static final String SPARK_YARN_TAGS = "spark.yarn.tags"; protected static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"}; http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java index 1e9d8af..f109318 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java @@ -65,6 +65,7 @@ public class ShellMain extends LauncherMain { Configuration actionConf = loadActionConf(); setYarnTag(actionConf); setApplicationTags(actionConf, TEZ_APPLICATION_TAGS); + setApplicationTags(actionConf, SPARK_YARN_TAGS); int exitCode = execute(actionConf); if (exitCode != 0) { http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index bf2869b..407ba4b 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -159,6 +159,11 @@ public class SparkMain extends LauncherMain { sparkArgs.add("--conf"); sparkArgs.add("spark.driver.extraJavaOptions=-Dlog4j.configuration=" + SPARK_LOG4J_PROPS); + if (actionConf.get(MAPREDUCE_JOB_TAGS) != null) { + sparkArgs.add("--conf"); + sparkArgs.add("spark.yarn.tags=" + actionConf.get(MAPREDUCE_JOB_TAGS)); + } + if (!addedHiveSecurityToken) { sparkArgs.add("--conf"); sparkArgs.add(HIVE_SECURITY_TOKEN + "=false");
