Repository: oozie Updated Branches: refs/heads/master ba7a7b85e -> ec70027da
OOZIE-2531 Prevent Spark trying for token which is already available (satishsaley via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ec70027d Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ec70027d Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ec70027d Branch: refs/heads/master Commit: ec70027da0012aa8a7dd0001a05e524c76a62b1c Parents: ba7a7b8 Author: Robert Kanter <[email protected]> Authored: Wed May 18 23:37:16 2016 -0700 Committer: Robert Kanter <[email protected]> Committed: Wed May 18 23:37:16 2016 -0700 ---------------------------------------------------------------------- release-log.txt | 1 + .../apache/oozie/action/hadoop/SparkMain.java | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/ec70027d/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 8ea3c4a..042a9f6 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2531 Prevent Spark trying for token which is already available (satishsaley via rkanter) OOZIE-2509 SLA job status can stuck in running state (puru) OOZIE-2529 Support adding secret keys to Credentials of Launcher (satishsaley via rohini) OOZIE-1402 Increase retry interval for non-progressing coordinator action with fix value (satishsaley via puru) http://git-wip-us.apache.org/repos/asf/oozie/blob/ec70027d/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 4587ab4..a09ecfb 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -37,6 +37,8 @@ public class SparkMain extends LauncherMain { private static final String DRIVER_CLASSPATH = "spark.driver.extraClassPath="; private static final String DIST_FILES = "spark.yarn.dist.files="; private static final String JARS_OPTION = "--jars"; + private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled"; + private static final String HBASE_SECURITY_TOKEN = "spark.yarn.security.tokens.hbase.enabled"; private String sparkJars = null; private String sparkClasspath = null; @@ -104,6 +106,8 @@ public class SparkMain extends LauncherMain { boolean addedDriverClasspath = false; boolean addedDistFiles = false; boolean addedJars = false; + boolean addedHiveSecurityToken = false; + boolean addedHBaseSecurityToken = false; String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS); if (StringUtils.isNotEmpty(sparkOpts)) { List<String> sparkOptions = splitSparkOpts(sparkOpts); @@ -135,6 +139,12 @@ public class SparkMain extends LauncherMain { addedDriverClasspath = true; } } + if (opt.startsWith(HIVE_SECURITY_TOKEN)) { + addedHiveSecurityToken = true; + } + if (opt.startsWith(HBASE_SECURITY_TOKEN)) { + addedHBaseSecurityToken = true; + } sparkArgs.add(opt); } } @@ -156,7 +166,14 @@ public class SparkMain extends LauncherMain { sparkArgs.add("--conf"); sparkArgs.add(DIST_FILES + sparkJars); } - + if (!addedHiveSecurityToken) { + sparkArgs.add("--conf"); + sparkArgs.add(HIVE_SECURITY_TOKEN + "=false"); + } + if (!addedHBaseSecurityToken) { + sparkArgs.add("--conf"); + sparkArgs.add(HBASE_SECURITY_TOKEN + "=false"); + } if (!sparkArgs.contains(VERBOSE_OPTION)) { sparkArgs.add(VERBOSE_OPTION); }
