Repository: tez Updated Branches: refs/heads/master 8126d2eb3 -> ae984705e
TEZ-3326. Display JVM system properties in AM and task logs. (Eric Badger via hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ae984705 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ae984705 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ae984705 Branch: refs/heads/master Commit: ae984705e01b357ba5b7fc972b58be4d75f5016d Parents: 8126d2e Author: Hitesh Shah <[email protected]> Authored: Thu Sep 1 14:13:49 2016 -0700 Committer: Hitesh Shah <[email protected]> Committed: Thu Sep 1 14:13:49 2016 -0700 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../java/org/apache/tez/common/TezUtils.java | 18 ++++++++++++++ .../apache/tez/dag/api/TezConfiguration.java | 19 ++++++++++++++ .../org/apache/tez/common/TestTezUtils.java | 26 ++++++++++++++++++++ .../org/apache/tez/dag/app/DAGAppMaster.java | 8 ++++++ .../org/apache/tez/runtime/task/TezChild.java | 9 +++++++ 6 files changed, 83 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index baade62..b73dd3f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3326. Display JVM system properties in AM and task logs. TEZ-3009. Errors that occur during container task acquisition are not logged. TEZ-2852. TestVertexImpl fails due to race in AsyncDispatcher. TEZ-3395. Refactor ShuffleVertexManager to make parts of it re-usable in other plugins. @@ -101,6 +102,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3326. Display JVM system properties in AM and task logs. TEZ-3009. Errors that occur during container task acquisition are not logged. TEZ-2852. TestVertexImpl fails due to race in AsyncDispatcher. TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent. @@ -586,6 +588,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3326. Display JVM system properties in AM and task logs. TEZ-3009. Errors that occur during container task acquisition are not logged. TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent. TEZ-3286. Allow clients to set processor reserved memory per vertex (instead of per container). http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-api/src/main/java/org/apache/tez/common/TezUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java index 93d373b..3aa1914 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java @@ -20,6 +20,7 @@ package org.apache.tez.common; import java.io.IOException; import java.io.OutputStream; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.tez.client.TezClientUtils; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.api.records.DAGProtos; @@ -66,6 +68,22 @@ public class TezUtils { TezClientUtils.addLog4jSystemProperties(logLevel, vargs); } + public static String getSystemPropertiesToLog(Configuration conf) { + Collection <String> keys = conf.getTrimmedStringCollection( + TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG); + if (keys.isEmpty()) { + keys = TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT; + } + StringBuilder sb = new StringBuilder(); + sb.append("\n/************************************************************\n"); + sb.append("[system properties]\n"); + for (String key : keys) { + sb.append(key).append(": ").append(System.getProperty(key)).append('\n'); + } + sb.append("************************************************************/"); + return sb.toString(); + } + /** * Convert a Configuration to compressed ByteString using Protocol buffer http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 15e937f..5aa8f7e 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -19,7 +19,11 @@ package org.apache.tez.dag.api; import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -1647,4 +1651,19 @@ public class TezConfiguration extends Configuration { public static final String TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES = TEZ_PREFIX + "am.ats.v15.override.summary-types"; public static final boolean TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT = true; + + /** + * String value. Determines what JVM properties will be logged for debugging purposes + * in the AM and Task runtime logs. + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG = + TEZ_PREFIX + "tez.jvm.system-properties-to-log"; + public static final List<String> TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT = + Collections.unmodifiableList(Arrays.asList( + "os.name","os.version","java.home","java.runtime.version", + "java.vendor","java.version","java.vm.name","java.class.path", + "java.io.tmpdir","user.dir","user.name")); + } http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java ---------------------------------------------------------------------- diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java index c88fa67..2eab776 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java @@ -17,11 +17,16 @@ package org.apache.tez.common; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.util.BitSet; import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.UserPayload; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -225,5 +230,26 @@ public class TestTezUtils { } + @Test + public void testLogSystemProperties() throws Exception { + Configuration conf = new Configuration(); + // test default logging + conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, " "); + String value = TezUtils.getSystemPropertiesToLog(conf); + for(String key: TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT) { + assertTrue(value.contains(key)); + } + + // test logging of selected keys + String classpath = "java.class.path"; + String os = "os.name"; + String version = "java.version"; + conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, classpath + ", " + os); + value = TezUtils.getSystemPropertiesToLog(conf); + assertNotNull(value); + assertTrue(value.contains(classpath)); + assertTrue(value.contains(os)); + assertFalse(value.contains(version)); + } } http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 7ca7118..cc07fb7 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2356,6 +2356,14 @@ public class DAGAppMaster extends AbstractService { ShutdownHookManager.get().addShutdownHook( new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY); + // log the system properties + if (LOG.isInfoEnabled()) { + String systemPropsToLog = TezUtils.getSystemPropertiesToLog(conf); + if (systemPropsToLog != null) { + LOG.info(systemPropsToLog); + } + } + initAndStartAppMaster(appMaster, conf); } catch (Throwable t) { http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java index 3c2d19a..2255ed7 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java @@ -54,6 +54,7 @@ import org.apache.tez.common.ContainerTask; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezLocalResource; import org.apache.tez.common.TezTaskUmbilicalProtocol; +import org.apache.tez.common.TezUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.common.counters.Limits; import org.apache.tez.common.security.JobTokenIdentifier; @@ -502,6 +503,14 @@ public class TezChild { HadoopShim hadoopShim = new HadoopShimsLoader(defaultConf).getHadoopShim(); + // log the system properties + if (LOG.isInfoEnabled()) { + String systemPropsToLog = TezUtils.getSystemPropertiesToLog(defaultConf); + if (systemPropsToLog != null) { + LOG.info(systemPropsToLog); + } + } + TezChild tezChild = newTezChild(defaultConf, host, port, containerIdentifier, tokenIdentifier, attemptNumber, localDirs, System.getenv(Environment.PWD.name()), System.getenv(), pid, new ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())),
