TEZ-3659. AM/Task classpath should not contain hadoop conf directory. (harishjp)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/293c5933 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/293c5933 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/293c5933 Branch: refs/heads/TEZ-1190 Commit: 293c5933f27be96091320f4a5deb80815db33cfd Parents: a339d05 Author: Harish JP <[email protected]> Authored: Fri Mar 17 08:06:41 2017 +0530 Committer: Harish JP <[email protected]> Committed: Fri Mar 17 08:07:55 2017 +0530 ---------------------------------------------------------------------- .../java/org/apache/tez/common/TezYARNUtils.java | 7 +++---- .../org/apache/tez/dag/api/TezConfiguration.java | 13 +++++++++++++ .../java/org/apache/tez/common/TestTezYARNUtils.java | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java index d5da672..bd6de11 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java @@ -18,7 +18,6 @@ package org.apache.tez.common; import java.io.File; -import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -103,10 +102,10 @@ public class TezYARNUtils { classpathBuilder.append(c.trim()) .append(File.pathSeparator); } - } else { + } else if (conf.getBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF, + TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT)) { // Setup HADOOP_CONF_DIR after PWD and tez-libs, if it's required. - classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()) - .append(File.pathSeparator); + classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()).append(File.pathSeparator); } if (!userClassesTakesPrecedence) { http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 5900f5e..40f84e6 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -1158,6 +1158,19 @@ public class TezConfiguration extends Configuration { TEZ_PREFIX + "cluster.additional.classpath.prefix"; /** + * Boolean value. + * If this value is true then tez explicitly adds hadoop conf directory into classpath for AM and + * task containers. Default is false. + */ + @Private + @Unstable + @ConfigurationScope(Scope.CLIENT) + @ConfigurationProperty(type="boolean") + public static final String TEZ_CLASSPATH_ADD_HADOOP_CONF = TEZ_PREFIX + + "classpath.add-hadoop-conf"; + public static final boolean TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT = false; + + /** * Session-related properties */ @Private http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java index 9f02801..4fdd61b 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java @@ -70,6 +70,21 @@ public class TestTezYARNUtils { Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*")); Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "lib" + File.separator + "*")); + Assert.assertTrue(!classpath.contains(Environment.HADOOP_CONF_DIR.$())); + Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) < + classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME)); + } + + @Test(timeout = 5000) + public void testNoHadoopConfInClasspath() { + Configuration conf = new Configuration(false); + conf.setBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF, true); + String classpath = TezYARNUtils.getFrameworkClasspath(conf, true); + Assert.assertTrue(classpath.contains(Environment.PWD.$())); + Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*")); + Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*")); + Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + + "lib" + File.separator + "*")); Assert.assertTrue(classpath.contains(Environment.HADOOP_CONF_DIR.$())); Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) < classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
