Author: vinodkv
Date: Sun Mar 16 18:32:05 2014
New Revision: 1578135
URL: http://svn.apache.org/r1578135
Log:
YARN-1824. Improved NodeManager and clients to be able to handle cross platform
application submissions. Contributed by Jian He.
MAPREDUCE-4052. Improved MapReduce clients to use NodeManagers' ability to
handle cross platform application submissions. Contributed by Jian He.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Sun Mar 16 18:32:05 2014
@@ -303,6 +303,9 @@ Release 2.4.0 - UNRELEASED
YARN-1658. Modified web-app framework to let standby RMs redirect
web-service calls to the active RM. (Cindy Li via vinodkv)
+ YARN-1824. Improved NodeManager and clients to be able to handle cross
+ platform application submissions. (Jian He via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
Sun Mar 16 18:32:05 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
@@ -59,6 +60,39 @@ public interface ApplicationConstants {
*/
public static final String LOG_DIR_EXPANSION_VAR = "<LOG_DIR>";
+ /**
+ * This constant is used to construct class path and it will be replaced with
+ * real class path separator(':' for Linux and ';' for Windows) by
+ * NodeManager on container launch. User has to use this constant to
construct
+ * class path if user wants cross-platform practice i.e. submit an
application
+ * from a Windows client to a Linux/Unix server or vice versa.
+ */
+ @Public
+ @Unstable
+ public static final String CLASS_PATH_SEPARATOR= "<CPS>";
+
+ /**
+ * The following two constants are used to expand parameter and it will be
+ * replaced with real parameter expansion marker ('%' for Windows and '$' for
+ * Linux) by NodeManager on container launch. For example: {{VAR}} will be
+ * replaced as $VAR on Linux, and %VAR% on Windows. User has to use this
+ * constant to construct class path if user wants cross-platform practice
i.e.
+ * submit an application from a Windows client to a Linux/Unix server or vice
+ * versa.
+ */
+ @Public
+ @Unstable
+ public static final String PARAMETER_EXPANSION_LEFT="{{";
+
+ /**
+ * User has to use this constant to construct class path if user wants
+ * cross-platform practice i.e. submit an application from a Windows client
to
+ * a Linux/Unix server or vice versa.
+ */
+ @Public
+ @Unstable
+ public static final String PARAMETER_EXPANSION_RIGHT="}}";
+
public static final String STDERR = "stderr";
public static final String STDOUT = "stdout";
@@ -206,7 +240,15 @@ public interface ApplicationConstants {
public String toString() {
return variable;
}
-
+
+ /**
+ * Expand the environment variable based on client OS environment variable
+ * expansion syntax (e.g. $VAR for Linux and %VAR% for Windows).
+ * <p>
+ * Note: Use $$() method for cross-platform practice i.e. submit an
+ * application from a Windows client to a Linux/Unix server or vice versa.
+ * </p>
+ */
public String $() {
if (Shell.WINDOWS) {
return "%" + variable + "%";
@@ -214,5 +256,18 @@ public interface ApplicationConstants {
return "$" + variable;
}
}
+
+ /**
+ * Expand the environment variable in platform-agnostic syntax. The
+ * parameter expansion marker "{{VAR}}" will be replaced with real
parameter
+ * expansion marker ('%' for Windows and '$' for Linux) by NodeManager on
+ * container launch. For example: {{VAR}} will be replaced as $VAR on
Linux,
+ * and %VAR% on Windows.
+ */
+ @Public
+ @Unstable
+ public String $$() {
+ return PARAMETER_EXPANSION_LEFT + variable + PARAMETER_EXPANSION_RIGHT;
+ }
}
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Sun Mar 16 18:32:05 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.HadoopIllegalAr
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
@@ -955,8 +956,39 @@ public class YarnConfiguration extends C
+ "application.classpath";
/**
- * Default CLASSPATH for YARN applications. A comma-separated list of
- * CLASSPATH entries
+ * Default platform-agnostic CLASSPATH for YARN applications. A
+ * comma-separated list of CLASSPATH entries. The parameter expansion marker
+ * will be replaced with real parameter expansion marker ('%' for Windows and
+ * '$' for Linux) by NodeManager on container launch. For example: {{VAR}}
+ * will be replaced as $VAR on Linux, and %VAR% on Windows.
+ */
+ @Public
+ @Unstable
+ public static final String[]
DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= {
+ ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(),
+ ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$()
+ + "/share/hadoop/common/*",
+ ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$()
+ + "/share/hadoop/common/lib/*",
+ ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$()
+ + "/share/hadoop/hdfs/*",
+ ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$()
+ + "/share/hadoop/hdfs/lib/*",
+ ApplicationConstants.Environment.HADOOP_YARN_HOME.$$()
+ + "/share/hadoop/yarn/*",
+ ApplicationConstants.Environment.HADOOP_YARN_HOME.$$()
+ + "/share/hadoop/yarn/lib/*" };
+ /**
+ * <p>
+ * Default platform-specific CLASSPATH for YARN applications. A
+ * comma-separated list of CLASSPATH entries constructed based on the client
+ * OS environment expansion syntax.
+ * </p>
+ * <p>
+ * Note: Use {@link DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
+ * cross-platform practice i.e. submit an application from a Windows client
to
+ * a Linux/Unix server or vice versa.
+ * </p>
*/
public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = {
ApplicationConstants.Environment.HADOOP_CONF_DIR.$(),
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
Sun Mar 16 18:32:05 2014
@@ -47,6 +47,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@@ -74,7 +76,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -85,6 +86,7 @@ import org.apache.hadoop.yarn.client.api
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -223,8 +225,9 @@ public class ApplicationMaster {
private long shellScriptPathLen = 0;
// Hardcoded path to shell script in launch container's local env
- private static final String ExecShellStringPath = "ExecShellScript.sh";
- private static final String ExecBatScripStringtPath = "ExecBatScript.bat";
+ private static final String ExecShellStringPath = Client.SCRIPT_PATH + ".sh";
+ private static final String ExecBatScripStringtPath = Client.SCRIPT_PATH
+ + ".bat";
// Hardcoded path to custom log_properties
private static final String log4jPath = "log4j.properties";
@@ -846,15 +849,29 @@ public class ApplicationMaster {
// In this scenario, if a shell script is specified, we need to have it
// copied and made available to the container.
if (!shellScriptPath.isEmpty()) {
+ Path renamedSchellScriptPath = null;
+ if (Shell.WINDOWS) {
+ renamedSchellScriptPath = new Path(shellScriptPath + ".bat");
+ } else {
+ renamedSchellScriptPath = new Path(shellScriptPath + ".sh");
+ }
+ try {
+ FileSystem fs = renamedSchellScriptPath.getFileSystem(conf);
+ fs.rename(new Path(shellScriptPath), renamedSchellScriptPath);
+ } catch (IOException e) {
+ LOG.warn("Not able to add suffix (.bat/.sh) to the shell script
filename");
+ throw new YarnRuntimeException(e);
+ }
+
LocalResource shellRsrc = Records.newRecord(LocalResource.class);
shellRsrc.setType(LocalResourceType.FILE);
shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
try {
shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI(
- shellScriptPath)));
+ renamedSchellScriptPath.toString())));
} catch (URISyntaxException e) {
LOG.error("Error when trying to use shell script path specified"
- + " in env, path=" + shellScriptPath);
+ + " in env, path=" + renamedSchellScriptPath);
e.printStackTrace();
// A failure scenario on bad input such as invalid shell script path
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
Sun Mar 16 18:32:05 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.applications.distributedshell;
-import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -49,7 +48,6 @@ import org.apache.hadoop.io.DataOutputBu
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -177,8 +175,7 @@ public class Client {
// Hardcoded path to custom log_properties
private static final String log4jPath = "log4j.properties";
- private static final String linuxShellPath = "ExecShellScript.sh";
- private static final String windowBatPath = "ExecBatScript.bat";
+ public static final String SCRIPT_PATH = "ExecScript";
/**
* @param args Command line arguments
@@ -492,8 +489,7 @@ public class Client {
if (!shellScriptPath.isEmpty()) {
Path shellSrc = new Path(shellScriptPath);
String shellPathSuffix =
- appName + "/" + appId.getId() + "/"
- + (Shell.WINDOWS ? windowBatPath : linuxShellPath);
+ appName + "/" + appId.getId() + "/" + SCRIPT_PATH;
Path shellDst =
new Path(fs.getHomeDirectory(), shellPathSuffix);
fs.copyFromLocalFile(false, true, shellSrc, shellDst);
@@ -535,15 +531,16 @@ public class Client {
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar
- StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$())
- .append(File.pathSeparatorChar).append("./*");
+ StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$())
+ .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
for (String c : conf.getStrings(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
- classPathEnv.append(File.pathSeparatorChar);
+ YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
+ classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
classPathEnv.append(c.trim());
}
- classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
+ classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append(
+ "./log4j.properties");
// add the runtime classpath needed for tests to work
if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
@@ -560,7 +557,7 @@ public class Client {
// Set java executable command
LOG.info("Setting up app master command");
- vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
+ vargs.add(Environment.JAVA_HOME.$$() + "/bin/java");
// Set Xmx based on am memory size
vargs.add("-Xmx" + amMemory + "m");
// Set class name
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Sun Mar 16 18:32:05 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringInterner;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -70,7 +71,7 @@ public class Apps {
}
public static void setEnvFromInputString(Map<String, String> env,
- String envString) {
+ String envString, String classPathSeparator) {
if (envString != null && envString.length() > 0) {
String childEnvs[] = envString.split(",");
Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex());
@@ -92,7 +93,7 @@ public class Apps {
m.appendReplacement(sb, Matcher.quoteReplacement(replace));
}
m.appendTail(sb);
- addToEnvironment(env, parts[0], sb.toString());
+ addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
}
}
}
@@ -101,14 +102,19 @@ public class Apps {
@Unstable
public static void addToEnvironment(
Map<String, String> environment,
- String variable, String value) {
+ String variable, String value, String classPathSeparator) {
String val = environment.get(variable);
if (val == null) {
val = value;
} else {
- val = val + File.pathSeparator + value;
+ val = val + classPathSeparator + value;
}
environment.put(StringInterner.weakIntern(variable),
StringInterner.weakIntern(val));
}
+
+ public static String crossPlatformify(String var) {
+ return ApplicationConstants.PARAMETER_EXPANSION_LEFT + var
+ + ApplicationConstants.PARAMETER_EXPANSION_RIGHT;
+ }
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
Sun Mar 16 18:32:05 2014
@@ -76,6 +76,8 @@ import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import com.google.common.annotations.VisibleForTesting;
+
public class ContainerLaunch implements Callable<Integer> {
private static final Log LOG = LogFactory.getLog(ContainerLaunch.class);
@@ -124,6 +126,25 @@ public class ContainerLaunch implements
YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
}
+ @VisibleForTesting
+ public static String expandEnvironment(String var,
+ Path containerLogDir) {
+ var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
+ containerLogDir.toString());
+ var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
+ File.pathSeparator);
+
+ // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
+ // as %VAR% and on Linux replaced as "$VAR"
+ if (Shell.WINDOWS) {
+ var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
+ } else {
+ var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
+ var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
+ }
+ return var;
+ }
+
@Override
@SuppressWarnings("unchecked") // dispatcher not typed
public Integer call() {
@@ -165,8 +186,7 @@ public class ContainerLaunch implements
dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
for (String str : command) {
// TODO: Should we instead work via symlinks without this grammar?
- newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
- containerLogDir.toString()));
+ newCmds.add(expandEnvironment(str, containerLogDir));
}
launchContext.setCommands(newCmds);
@@ -174,11 +194,8 @@ public class ContainerLaunch implements
// Make a copy of env to iterate & do variable expansion
for (Entry<String, String> entry : environment.entrySet()) {
String value = entry.getValue();
- entry.setValue(
- value.replace(
- ApplicationConstants.LOG_DIR_EXPANSION_VAR,
- containerLogDir.toString())
- );
+ value = expandEnvironment(value, containerLogDir);
+ entry.setValue(value);
}
// /////////////////////////// End of variable expansion
@@ -647,12 +664,9 @@ public class ContainerLaunch implements
}
// variables here will be forced in, even if the container has specified
them.
- Apps.setEnvFromInputString(
- environment,
- conf.get(
- YarnConfiguration.NM_ADMIN_USER_ENV,
- YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV)
- );
+ Apps.setEnvFromInputString(environment, conf.get(
+ YarnConfiguration.NM_ADMIN_USER_ENV,
+ YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV), File.pathSeparator);
// TODO: Remove Windows check and use this approach on all platforms after
// additional testing. See YARN-358.
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
Sun Mar 16 18:32:05 2014
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.no
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.spy;
import java.io.BufferedReader;
import java.io.File;
@@ -48,6 +47,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -73,12 +73,12 @@ import org.apache.hadoop.yarn.security.C
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
-import
org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
@@ -287,6 +287,31 @@ public class TestContainerLaunch extends
}
}
+ @Test(timeout = 10000)
+ public void testEnvExpansion() throws IOException {
+ Path logPath = new Path("/nm/container/logs");
+ String input =
+ Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
+ + ApplicationConstants.CLASS_PATH_SEPARATOR
+ + Apps.crossPlatformify("HADOOP_HOME") +
"/share/hadoop/common/lib/*"
+ + ApplicationConstants.CLASS_PATH_SEPARATOR
+ + Apps.crossPlatformify("HADOOP_LOG_HOME")
+ + ApplicationConstants.LOG_DIR_EXPANSION_VAR;
+
+ String res = ContainerLaunch.expandEnvironment(input, logPath);
+
+ if (Shell.WINDOWS) {
+ Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
+ + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
+ + "%HADOOP_LOG_HOME%/nm/container/logs", res);
+ } else {
+ Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
+ + "$HADOOP_HOME/share/hadoop/common/lib/*:"
+ + "$HADOOP_LOG_HOME/nm/container/logs", res);
+ }
+ System.out.println(res);
+ }
+
@Test (timeout = 20000)
public void testContainerLaunchStdoutAndStderrDiagnostics() throws
IOException {
