Repository: tez Updated Branches: refs/heads/master b4be3619a -> 43f7b5e3a
TEZ-3466. Tez classpath building to mimic mapreduce classpath building (jlowe) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/43f7b5e3 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/43f7b5e3 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/43f7b5e3 Branch: refs/heads/master Commit: 43f7b5e3a6b85e3e4fbb86c3bed81e5b586c79dd Parents: b4be361 Author: Jason Lowe <jl...@apache.org> Authored: Thu Oct 13 22:22:07 2016 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Thu Oct 13 22:22:07 2016 +0000 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/tez/common/TezYARNUtils.java | 36 +++++++++++++------- .../apache/tez/dag/api/TezConfiguration.java | 15 ++++++-- .../org/apache/tez/common/TestTezYARNUtils.java | 13 +++++++ .../tez/mapreduce/hadoop/DeprecatedKeys.java | 4 +++ .../mapreduce/hadoop/TestDeprecatedKeys.java | 6 ++++ 6 files changed, 61 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 79b0e38..ef6b890 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3466. Tez classpath building to mimic mapreduce classpath building. TEZ-3453. Correct the downloaded ATS dag data location for analyzer. TEZ-3449. Fix Spelling typos. TEZ-3464. Fix findbugs warnings in tez-dag mainLoop http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java index c505ca8..145f928 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java @@ -43,20 +43,13 @@ public class TezYARNUtils { public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) { StringBuilder classpathBuilder = new StringBuilder(); - - // Add any additional user-specified classpath - String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX); - if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) { - classpathBuilder.append(additionalClasspath) - .append(File.pathSeparator); + boolean userClassesTakesPrecedence = + conf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, + TezConfiguration.TEZ_USER_CLASSPATH_FIRST_DEFAULT); + if (userClassesTakesPrecedence) { + addUserSpecifiedClasspath(classpathBuilder, conf); } - // Add PWD:PWD/* - classpathBuilder.append(Environment.PWD.$()) - .append(File.pathSeparator) - .append(Environment.PWD.$() + File.separator + "*") - .append(File.pathSeparator); - String [] tezLibUrisClassPath = conf.getStrings(TezConfiguration.TEZ_LIB_URIS_CLASSPATH); if(!conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false) && @@ -107,10 +100,29 @@ public class TezYARNUtils { .append(File.pathSeparator); } + if (!userClassesTakesPrecedence) { + addUserSpecifiedClasspath(classpathBuilder, conf); + } String classpath = classpathBuilder.toString(); return StringInterner.weakIntern(classpath); } + private static void addUserSpecifiedClasspath(StringBuilder classpathBuilder, + Configuration conf) { + // Add any additional user-specified classpath + String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX); + if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) { + classpathBuilder.append(additionalClasspath) + .append(File.pathSeparator); + } + + // Add PWD:PWD/* + classpathBuilder.append(Environment.PWD.$()) + .append(File.pathSeparator) + .append(Environment.PWD.$() + File.separator + "*") + .append(File.pathSeparator); + } + public static void appendToEnvFromInputString(Map<String, String> env, String envString, String classPathSeparator) { if (envString != null && envString.length() > 0) { http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 5aa8f7e..77ea4ff 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -1094,11 +1094,22 @@ public class TezConfiguration extends Configuration { public static final boolean TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT = false; /** + * Boolean value. + * Specify whether the user classpath takes precedence over the Tez framework + * classpath. + */ + @ConfigurationScope(Scope.CLIENT) + @ConfigurationProperty(type="boolean") + public static final String TEZ_USER_CLASSPATH_FIRST = TEZ_PREFIX + "user.classpath.first"; + public static final boolean TEZ_USER_CLASSPATH_FIRST_DEFAULT = true; + + /** * String value. * * Specify additional classpath information to be used for Tez AM and all containers. - * This will be prepended to the classpath before all framework specific components have been - * specified. + * If {@link #TEZ_USER_CLASSPATH_FIRST} is true then this will be added to the classpath + * before all framework specific components have been specified, otherwise this will + * be added after the framework specific components. */ @ConfigurationScope(Scope.AM) @ConfigurationProperty http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java index 2dabf51..092bd50 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java @@ -43,6 +43,19 @@ public class TestTezYARNUtils { classpath.indexOf(Environment.PWD.$())); } + @Test(timeout = 20000) + public void testUserClasspathFirstFalse() { + Configuration conf = new Configuration(false); + conf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false); + conf.set(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX, "foobar"); + String classpath = TezYARNUtils.getFrameworkClasspath(conf, true); + Assert.assertTrue(classpath.contains("foobar")); + Assert.assertTrue(classpath.indexOf("foobar") > + classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME)); + Assert.assertTrue(classpath.indexOf("foobar") > + classpath.indexOf(Environment.PWD.$())); + } + @Test(timeout = 5000) public void testBasicArchiveClasspath() { Configuration conf = new Configuration(false); http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index a826f6d..9ae58c0 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -86,6 +86,8 @@ public class DeprecatedKeys { TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS); mrParamToDAGParamMap.put(MRJobConfig.JOB_TAGS, TezConfiguration.TEZ_APPLICATION_TAGS); + mrParamToDAGParamMap.put(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, + TezConfiguration.TEZ_USER_CLASSPATH_FIRST); } // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. @@ -157,6 +159,8 @@ public class DeprecatedKeys { registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS); registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC); + + registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, TezConfiguration.TEZ_USER_CLASSPATH_FIRST); } private static void addDeprecatedKeys() { http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java index ae218a0..c574e38 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java @@ -43,6 +43,7 @@ public class TestDeprecatedKeys { jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f); jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true); jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f); + jobConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); MRHelpers.translateMRConfToTez(jobConf); @@ -64,6 +65,7 @@ public class TestDeprecatedKeys { assertEquals(0.33f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0), 0.01f); + assertEquals(false, jobConf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, true)); } @Test(timeout = 5000) @@ -76,6 +78,7 @@ public class TestDeprecatedKeys { jobConf.setInt(MRJobConfig.IO_SORT_MB, 100); jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100); jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f); + jobConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000); jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200); @@ -100,6 +103,7 @@ public class TestDeprecatedKeys { jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter"); jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator"); jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator"); + jobConf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false); jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false); jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true); @@ -131,6 +135,7 @@ public class TestDeprecatedKeys { assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "")); assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false)); assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0.0f), 0.0f); + assertEquals(false, jobConf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, true)); assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD)); assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES)); @@ -156,6 +161,7 @@ public class TestDeprecatedKeys { assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS)); assertNull(jobConf.get("map.sort.class")); assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART)); + assertNull(jobConf.get(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST)); } }