Repository: hadoop Updated Branches: refs/heads/branch-2 2e99210e4 -> 6044b5e5f
MAPREDUCE-5232. Add a configuration to be able to log classpath and other system properties on mapreduce JVMs startup. Contributed by Sangjin Lee. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1482643 13f79535-47bb-0310-9956-ffa450edef68 (cherry picked from commit 4d8e350750748b919ee2158690a44cd9fd80dcae) Conflicts: hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6044b5e5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6044b5e5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6044b5e5 Branch: refs/heads/branch-2 Commit: 6044b5e5f6402ddcc163d9087af6228f24291fd2 Parents: 2e99210 Author: Vinod Kumar Vavilapalli <vino...@apache.org> Authored: Tue May 14 23:43:53 2013 +0000 Committer: Gera Shegalov <g...@apache.org> Committed: Thu Jun 4 11:15:30 2015 -0700 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../org/apache/hadoop/mapred/YarnChild.java | 6 ++++ .../hadoop/mapreduce/v2/app/MRAppMaster.java | 6 ++++ .../apache/hadoop/mapreduce/v2/util/MRApps.java | 31 ++++++++++++++++++++ .../hadoop/mapreduce/v2/util/TestMRApps.java | 21 ++++++++++++- .../apache/hadoop/mapreduce/MRJobConfig.java | 5 ++++ .../src/main/resources/mapred-default.xml | 6 ++++ 7 files changed, 77 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index cf40293..d215fc2 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -189,6 +189,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6382. Don't escape HTML links in Diagnostics in JHS job overview. (Siqi Li via gera) + MAPREDUCE-5232. Add a configuration to be able to log classpath and other + system properties on mapreduce JVMs startup. (Sangjin Lee via vinodkv) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index 7f3111f..ea9733c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -141,6 +141,12 @@ class YarnChild { // Create the job-conf and set credentials configureTask(job, task, credentials, jt); + // log the system properties + String systemPropsToLog = MRApps.getSystemPropertiesToLog(job); + if (systemPropsToLog != null) { + LOG.info(systemPropsToLog); + } + // Initiate Java VM metrics JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId()); childUGI = UserGroupInformation.createRemoteUser(System http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 99c5822..942e25b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -1495,6 +1495,12 @@ public class MRAppMaster extends CompositeService { conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE)); MRWebAppUtil.initialize(conf); + // log the system properties + String systemPropsToLog = MRApps.getSystemPropertiesToLog(conf); + if (systemPropsToLog != null) { + LOG.info(systemPropsToLog); + } + String jobUserName = System .getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 7d1a1b5..40ba613 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -733,4 +733,35 @@ public class MRApps extends Apps { MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM); return crossPlatform ? env.$$() : env.$(); } + + /** + * Return lines for system property keys and values per configuration. + * + * @return the formatted string for the system property lines or null if no + * properties are specified. + */ + public static String getSystemPropertiesToLog(Configuration conf) { + String key = conf.get(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG, + MRJobConfig.DEFAULT_MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG); + if (key != null) { + key = key.trim(); // trim leading and trailing whitespace from the config + if (!key.isEmpty()) { + String[] props = key.split(","); + if (props.length > 0) { + StringBuilder sb = new StringBuilder(); + sb.append("\n/************************************************************\n"); + sb.append("[system properties]\n"); + for (String prop: props) { + prop = prop.trim(); // trim leading and trailing whitespace + if (!prop.isEmpty()) { + sb.append(prop).append(": ").append(System.getProperty(prop)).append('\n'); + } + } + sb.append("************************************************************/"); + return sb.toString(); + } + } + } + return null; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 2e77f5e..b233c84 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -480,7 +480,26 @@ public class TestMRApps { } public void initialize(URI name, Configuration conf) throws IOException {} } - + + @Test + public void testLogSystemProperties() throws Exception { + Configuration conf = new Configuration(); + // test no logging + conf.set(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG, " "); + String value = MRApps.getSystemPropertiesToLog(conf); + assertNull(value); + + // test logging of selected keys + String classpath = "java.class.path"; + String os = "os.name"; + String version = "java.version"; + conf.set(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG, classpath + ", " + os); + value = MRApps.getSystemPropertiesToLog(conf); + assertNotNull(value); + assertTrue(value.contains(classpath)); + assertTrue(value.contains(os)); + assertFalse(value.contains(version)); + } @Test public void testTaskStateUI() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 61d1ce7..202f860 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -205,6 +205,11 @@ public interface MRJobConfig { public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes"; + public static final String MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG = "mapreduce.jvm.system-properties-to-log"; + public static final String DEFAULT_MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG = + "os.name,os.version,java.home,java.runtime.version,java.vendor," + + "java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name"; + public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor"; public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 1b109f8..9de2c4e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1981,6 +1981,12 @@ </description> </property> +<property> + <name>mapreduce.jvm.system-properties-to-log</name> + <value>os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name</value> + <description>Comma-delimited list of system properties to log on mapreduce JVM start</description> +</property> + <!-- jobhistory properties --> <property>
