Repository: tez Updated Branches: refs/heads/branch-0.7 28ef31749 -> 34cb4f24a
TEZ-2679. Admin forms of launch env settings (jeagles) (cherry picked from commit 19808b7cd80f267a3474102cd0f761155a6df239) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/34cb4f24 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/34cb4f24 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/34cb4f24 Branch: refs/heads/branch-0.7 Commit: 34cb4f24af551493ff00794fe71448547fdaee2a Parents: 28ef317 Author: Jonathan Eagles <[email protected]> Authored: Thu Oct 29 11:22:42 2015 -0500 Committer: Jonathan Eagles <[email protected]> Committed: Thu Oct 29 11:25:45 2015 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/tez/client/TezClientUtils.java | 10 ++++-- .../org/apache/tez/common/TezYARNUtils.java | 20 ++++++------ .../main/java/org/apache/tez/dag/api/DAG.java | 5 ++- .../apache/tez/dag/api/TezConfiguration.java | 32 ++++++++++++++++++-- .../org/apache/tez/common/TestTezYARNUtils.java | 23 ++++++++++++++ 6 files changed, 75 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/34cb4f24/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index bb45e54..487f4d9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,7 @@ Apache Tez Change Log Release 0.7.1: Unreleased INCOMPATIBLE CHANGES + TEZ-2679. Admin forms of launch env settings ALL CHANGES TEZ-2900. Ignore V_INPUT_DATA_INFORMATION when vertex is in Failed/Killed/Error http://git-wip-us.apache.org/repos/asf/tez/blob/34cb4f24/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index f0e67c7..cf49c31 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -504,9 +504,13 @@ public class TezClientUtils { } Map<String, String> environment = new TreeMap<String, String>(); - TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV, - TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, tezLrsAsArchive); - + TezYARNUtils.setupDefaultEnv(environment, conf, + TezConfiguration.TEZ_AM_LAUNCH_ENV, + TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, + TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV, + TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT, + tezLrsAsArchive); + addVersionInfoToEnv(environment, apiVersionInfo); addLogParamsToEnv(environment, amLogParams); http://git-wip-us.apache.org/repos/asf/tez/blob/34cb4f24/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java index fc30780..d7093db 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java @@ -167,8 +167,8 @@ public class TezYARNUtils { } } - public static void setupDefaultEnv(Map<String, String> env, - Configuration conf, String confEnvKey, String confEnvKeyDefault, boolean usingArchive) { + public static void setupDefaultEnv(Map<String, String> env, Configuration conf, String userEnvKey, String userEnvDefault, + String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) { // Setup the CLASSPATH in environment // i.e. add { Hadoop jars, job jar, CWD } to classpath. String classpath = getFrameworkClasspath(conf, usingArchive); @@ -176,14 +176,16 @@ public class TezYARNUtils { ApplicationConstants.Environment.CLASSPATH.name(), classpath, File.pathSeparator); + // Pre-pend pwd to LD_LIBRARY_PATH + // Done separately here because this is known to work platform + // independent + TezYARNUtils.addToEnvironment(env, + Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator); + TezYARNUtils.appendToEnvFromInputString(env, + conf.get(userEnvKey, userEnvDefault), File.pathSeparator); // set any env from config if it is not set already - TezYARNUtils.setEnvIfAbsentFromInputString(env, conf.get( - confEnvKey, confEnvKeyDefault)); - - // Append pwd to LD_LIBRARY_PATH - // Done separately here because this is known to work platform independent - TezYARNUtils.addToEnvironment(env, Environment.LD_LIBRARY_PATH.name(), - Environment.PWD.$(), File.pathSeparator); + TezYARNUtils.appendToEnvFromInputString(env, + conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator); } public static void replaceInEnv(Map<String, String> env, String key, String value) { http://git-wip-us.apache.org/repos/asf/tez/blob/34cb4f24/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java index be75dda..71ea05d 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java @@ -868,7 +868,10 @@ public class DAG { Map<String, String> taskEnv = Maps.newHashMap(vertex.getTaskEnvironment()); TezYARNUtils.setupDefaultEnv(taskEnv, tezConf, TezConfiguration.TEZ_TASK_LAUNCH_ENV, - TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT, tezLrsAsArchive); + TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT, + TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_ENV, + TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT, + tezLrsAsArchive); for (Map.Entry<String, String> entry : taskEnv.entrySet()) { PlanKeyValuePair.Builder envSettingBuilder = PlanKeyValuePair.newBuilder(); envSettingBuilder.setKey(entry.getKey()); http://git-wip-us.apache.org/repos/asf/tez/blob/34cb4f24/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 9597a0d..d766507 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -340,17 +340,42 @@ public class TezConfiguration extends Configuration { "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin": "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_COMMON_HOME/lib/native/"; + /** + * String value. Env settings will be merged with {@link #TEZ_AM_LAUNCH_ENV} + * during the launch of the AppMaster process. This property will typically be configured to + * include default system env meant to be used by all jobs in a cluster. If required, the values can + * be appended to per job. + */ + @ConfigurationScope(Scope.AM) + public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV = + TEZ_AM_PREFIX + "launch.cluster-default.env"; + public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT = + NATIVE_LIB_PARAM_DEFAULT; + /** String value. Env settings for the Tez AppMaster process. * Should be specified as a comma-separated of key-value pairs where each pair * is defined as KEY=VAL * e.g. "LD_LIBRARY_PATH=.,USERNAME=foo" * These take least precedence compared to other methods of setting env. * These get added to the app master environment prior to launching it. - */ + * This setting will prepend existing settings in the cluster default + */ @ConfigurationScope(Scope.AM) public static final String TEZ_AM_LAUNCH_ENV = TEZ_AM_PREFIX + "launch.env"; - public static final String TEZ_AM_LAUNCH_ENV_DEFAULT = NATIVE_LIB_PARAM_DEFAULT; + public static final String TEZ_AM_LAUNCH_ENV_DEFAULT = ""; + + /** + * String value. Env settings will be merged with {@link #TEZ_TASK_LAUNCH_ENV} + * during the launch of the task process. This property will typically be configured to + * include default system env meant to be used by all jobs in a cluster. If required, the values can + * be appended to per job. + */ + @ConfigurationScope(Scope.VERTEX) + public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_ENV = + TEZ_TASK_PREFIX + "launch.cluster-default.env"; + public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT = + NATIVE_LIB_PARAM_DEFAULT; /** String value. Env settings for the Tez Task processes. * Should be specified as a comma-separated of key-value pairs where each pair @@ -358,11 +383,12 @@ public class TezConfiguration extends Configuration { * e.g. "LD_LIBRARY_PATH=.,USERNAME=foo" * These take least precedence compared to other methods of setting env * These get added to the task environment prior to launching it. + * This setting will prepend existing settings in the cluster default */ @ConfigurationScope(Scope.VERTEX) public static final String TEZ_TASK_LAUNCH_ENV = TEZ_TASK_PREFIX + "launch.env"; - public static final String TEZ_TASK_LAUNCH_ENV_DEFAULT = NATIVE_LIB_PARAM_DEFAULT; + public static final String TEZ_TASK_LAUNCH_ENV_DEFAULT = ""; @Private @ConfigurationScope(Scope.DAG) http://git-wip-us.apache.org/repos/asf/tez/blob/34cb4f24/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java index 6534cb6..6e9e06c 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java @@ -19,6 +19,8 @@ package org.apache.tez.common; import java.io.File; +import java.util.Map; +import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; @@ -57,4 +59,25 @@ public class TestTezYARNUtils { classpath.indexOf(Environment.HADOOP_CONF_DIR.$())); } + @Test(timeout = 5000) + public void testSetupDefaultEnvironment() { + Configuration conf = new Configuration(false); + conf.set(TezConfiguration.TEZ_AM_LAUNCH_ENV, "LD_LIBRARY_PATH=USER_PATH,USER_KEY=USER_VALUE"); + conf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV, "LD_LIBRARY_PATH=DEFAULT_PATH,DEFAULT_KEY=DEFAULT_VALUE"); + + Map<String, String> environment = new TreeMap<String, String>(); + TezYARNUtils.setupDefaultEnv(environment, conf, + TezConfiguration.TEZ_AM_LAUNCH_ENV, + TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, + TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV, + TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT, false); + + String value1 = environment.get("USER_KEY"); + Assert.assertEquals("User env should merge with default env", "USER_VALUE", value1); + String value2 = environment.get("DEFAULT_KEY"); + Assert.assertEquals("User env should merge with default env", "DEFAULT_VALUE", value2); + String value3 = environment.get("LD_LIBRARY_PATH"); + Assert.assertEquals("User env should append default env", + Environment.PWD.$() + File.pathSeparator + "USER_PATH" + File.pathSeparator + "DEFAULT_PATH", value3); + } }
