OOZIE-3036 Spark 2.2.0 support: tell Spark not to get any delegation tokens (andras.piros via gezapeti)
(cherry picked from commit 04a6d0541e83ab457db7c79e86986bb69a8eb432) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/169a1a2a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/169a1a2a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/169a1a2a Branch: refs/heads/branch-4.3 Commit: 169a1a2a18fea0f16cc278eef67b1e5e36cc2487 Parents: d64a4c3 Author: Gezapeti Cseh <[email protected]> Authored: Tue Aug 15 11:45:39 2017 +0200 Committer: satishsaley <[email protected]> Committed: Fri Dec 8 16:34:56 2017 -0800 ---------------------------------------------------------------------- release-log.txt | 1 + .../oozie/action/hadoop/SparkArgsExtractor.java | 68 ++++++++++++++++---- .../action/hadoop/TestSparkArgsExtractor.java | 20 ++++++ 3 files changed, 77 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/169a1a2a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 557f657..16d3af8 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.1 release +OOZIE-3036 Spark 2.2.0 support: tell Spark not to get any delegation tokens (andras.piros via gezapeti) OOZIE-3028 Oozie Pig Action fails with no python dependencies (dbist13 via rohini) OOZIE-2984 Parse spark-defaults.conf values with spaces without needing the quotes (andras.piros via gezapeti) OOZIE-2825 Custom Authentication doc page is not well formatted (Jan Hentschel via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/169a1a2a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java index 3270eea..a465d6d 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java @@ -40,8 +40,12 @@ class SparkArgsExtractor { private static final String FILES_OPTION = "--files"; private static final String ARCHIVES_OPTION = "--archives"; private static final String LOG4J_CONFIGURATION_JAVA_OPTION = "-Dlog4j.configuration="; - private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled"; - private static final String HBASE_SECURITY_TOKEN = "spark.yarn.security.tokens.hbase.enabled"; + private static final String SECURITY_TOKENS_HADOOPFS = "spark.yarn.security.tokens.hadoopfs.enabled"; + private static final String SECURITY_TOKENS_HIVE = "spark.yarn.security.tokens.hive.enabled"; + private static final String SECURITY_TOKENS_HBASE = "spark.yarn.security.tokens.hbase.enabled"; + private static final String SECURITY_CREDENTIALS_HADOOPFS = "spark.yarn.security.credentials.hadoopfs.enabled"; + private static final String SECURITY_CREDENTIALS_HIVE = "spark.yarn.security.credentials.hive.enabled"; + private static final String SECURITY_CREDENTIALS_HBASE = "spark.yarn.security.credentials.hbase.enabled"; private static final String PWD = "$PWD" + File.separator + "*"; private static final String MASTER_OPTION = "--master"; private static final String MODE_OPTION = "--deploy-mode"; @@ -114,8 +118,15 @@ class SparkArgsExtractor { if (jarPath != null && jarPath.endsWith(".py")) { pySpark = true; } - boolean addedHiveSecurityToken = false; - boolean addedHBaseSecurityToken = false; + + boolean addedSecurityTokensHadoopFS = false; + boolean addedSecurityTokensHive = false; + boolean addedSecurityTokensHBase = false; + + boolean addedSecurityCredentialsHadoopFS = false; + boolean addedSecurityCredentialsHive = false; + boolean addedSecurityCredentialsHBase = false; + boolean addedLog4jDriverSettings = false; boolean addedLog4jExecutorSettings = false; final StringBuilder driverClassPath = new StringBuilder(); @@ -145,12 +156,27 @@ class SparkArgsExtractor { addToSparkArgs = false; } } - if (opt.startsWith(HIVE_SECURITY_TOKEN)) { - addedHiveSecurityToken = true; + + if (opt.startsWith(SECURITY_TOKENS_HADOOPFS)) { + addedSecurityTokensHadoopFS = true; + } + if (opt.startsWith(SECURITY_TOKENS_HIVE)) { + addedSecurityTokensHive = true; } - if (opt.startsWith(HBASE_SECURITY_TOKEN)) { - addedHBaseSecurityToken = true; + if (opt.startsWith(SECURITY_TOKENS_HBASE)) { + addedSecurityTokensHBase = true; } + + if (opt.startsWith(SECURITY_CREDENTIALS_HADOOPFS)) { + addedSecurityCredentialsHadoopFS = true; + } + if (opt.startsWith(SECURITY_CREDENTIALS_HIVE)) { + addedSecurityCredentialsHive = true; + } + if (opt.startsWith(SECURITY_CREDENTIALS_HBASE)) { + addedSecurityCredentialsHBase = true; + } + if (opt.startsWith(EXECUTOR_EXTRA_JAVA_OPTIONS) || opt.startsWith(DRIVER_EXTRA_JAVA_OPTIONS)) { if (!opt.contains(LOG4J_CONFIGURATION_JAVA_OPTION)) { opt += " " + LOG4J_CONFIGURATION_JAVA_OPTION + SparkMain.SPARK_LOG4J_PROPS; @@ -222,14 +248,32 @@ class SparkArgsExtractor { sparkArgs.add(SPARK_YARN_TAGS + OPT_SEPARATOR + actionConf.get(LauncherMain.MAPREDUCE_JOB_TAGS)); } - if (!addedHiveSecurityToken) { + if (!addedSecurityTokensHadoopFS) { sparkArgs.add(CONF_OPTION); - sparkArgs.add(HIVE_SECURITY_TOKEN + OPT_SEPARATOR + Boolean.toString(false)); + sparkArgs.add(SECURITY_TOKENS_HADOOPFS + OPT_SEPARATOR + Boolean.toString(false)); } - if (!addedHBaseSecurityToken) { + if (!addedSecurityTokensHive) { sparkArgs.add(CONF_OPTION); - sparkArgs.add(HBASE_SECURITY_TOKEN + OPT_SEPARATOR + Boolean.toString(false)); + sparkArgs.add(SECURITY_TOKENS_HIVE + OPT_SEPARATOR + Boolean.toString(false)); } + if (!addedSecurityTokensHBase) { + sparkArgs.add(CONF_OPTION); + sparkArgs.add(SECURITY_TOKENS_HBASE + OPT_SEPARATOR + Boolean.toString(false)); + } + + if (!addedSecurityCredentialsHadoopFS) { + sparkArgs.add(CONF_OPTION); + sparkArgs.add(SECURITY_CREDENTIALS_HADOOPFS + OPT_SEPARATOR + Boolean.toString(false)); + } + if (!addedSecurityCredentialsHive) { + sparkArgs.add(CONF_OPTION); + sparkArgs.add(SECURITY_CREDENTIALS_HIVE + OPT_SEPARATOR + Boolean.toString(false)); + } + if (!addedSecurityCredentialsHBase) { + sparkArgs.add(CONF_OPTION); + sparkArgs.add(SECURITY_CREDENTIALS_HBASE + OPT_SEPARATOR + Boolean.toString(false)); + } + if (!addedLog4jExecutorSettings) { sparkArgs.add(CONF_OPTION); sparkArgs.add(EXECUTOR_EXTRA_JAVA_OPTIONS + LOG4J_CONFIGURATION_JAVA_OPTION + SparkMain.SPARK_LOG4J_PROPS); http://git-wip-us.apache.org/repos/asf/oozie/blob/169a1a2a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java index d0541ca..574bf24 100644 --- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java +++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java @@ -73,8 +73,12 @@ public class TestSparkArgsExtractor { "--conf", "user.property.after.spark.driver.extraClassPath=ddd", "--conf", "spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError " + "-XX:HeapDumpPath=/tmp -Dlog4j.configuration=spark-log4j.properties", + "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false", "--conf", "spark.yarn.security.tokens.hive.enabled=false", "--conf", "spark.yarn.security.tokens.hbase.enabled=false", + "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false", + "--conf", "spark.yarn.security.credentials.hive.enabled=false", + "--conf", "spark.yarn.security.credentials.hbase.enabled=false", "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties", "--verbose", "/lib/test.jar", @@ -113,8 +117,12 @@ public class TestSparkArgsExtractor { "-XX:HeapDumpPath=/tmp -Dlog4j.configuration=spark-log4j.properties", "--conf", "spark.executor.extraClassPath=aaa:$PWD/*", "--conf", "spark.driver.extraClassPath=ccc:$PWD/*", + "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false", "--conf", "spark.yarn.security.tokens.hive.enabled=false", "--conf", "spark.yarn.security.tokens.hbase.enabled=false", + "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false", + "--conf", "spark.yarn.security.credentials.hive.enabled=false", + "--conf", "spark.yarn.security.credentials.hbase.enabled=false", "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties", "--files", "spark-log4j.properties,hive-site.xml", "--conf", "spark.yarn.jar=null", @@ -148,8 +156,12 @@ public class TestSparkArgsExtractor { "--class", "org.apache.oozie.example.SparkFileCopy", "--conf", "spark.executor.extraClassPath=$PWD/*", "--conf", "spark.driver.extraClassPath=$PWD/*", + "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false", "--conf", "spark.yarn.security.tokens.hive.enabled=false", "--conf", "spark.yarn.security.tokens.hbase.enabled=false", + "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false", + "--conf", "spark.yarn.security.credentials.hive.enabled=false", + "--conf", "spark.yarn.security.credentials.hbase.enabled=false", "--conf", "spark.executor.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties", "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties", "--files", "spark-log4j.properties,hive-site.xml,aaa,ccc", @@ -182,8 +194,12 @@ public class TestSparkArgsExtractor { "--class", "org.apache.oozie.example.SparkFileCopy", "--conf", "spark.executor.extraClassPath=$PWD/*", "--conf", "spark.driver.extraClassPath=aaa:$PWD/*", + "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false", "--conf", "spark.yarn.security.tokens.hive.enabled=false", "--conf", "spark.yarn.security.tokens.hbase.enabled=false", + "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false", + "--conf", "spark.yarn.security.credentials.hive.enabled=false", + "--conf", "spark.yarn.security.credentials.hbase.enabled=false", "--conf", "spark.executor.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties", "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties", "--files", "spark-log4j.properties,hive-site.xml", @@ -221,8 +237,12 @@ public class TestSparkArgsExtractor { "-Dlog4j.configuration=spark-log4j.properties", "--conf", "spark.executor.extraClassPath=$PWD/*", "--conf", "spark.driver.extraClassPath=$PWD/*", + "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false", "--conf", "spark.yarn.security.tokens.hive.enabled=false", "--conf", "spark.yarn.security.tokens.hbase.enabled=false", + "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false", + "--conf", "spark.yarn.security.credentials.hive.enabled=false", + "--conf", "spark.yarn.security.credentials.hbase.enabled=false", "--files", "spark-log4j.properties,hive-site.xml", "--conf", "spark.yarn.jar=null", "--verbose",
