Repository: tez
Updated Branches:
refs/heads/branch-0.8 b23e17c6f -> b6b04de46
TEZ-3326. Display JVM system properties in AM and task logs. (Eric Badger via
hitesh)
(cherry picked from commit ae984705e01b357ba5b7fc972b58be4d75f5016d)
Conflicts:
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b6b04de4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b6b04de4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b6b04de4
Branch: refs/heads/branch-0.8
Commit: b6b04de4627f096f6b182cb44185158efdb6d561
Parents: b23e17c
Author: Hitesh Shah <[email protected]>
Authored: Thu Sep 1 14:13:49 2016 -0700
Committer: Hitesh Shah <[email protected]>
Committed: Thu Sep 1 14:18:46 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/common/TezUtils.java | 18 ++++++++++++++
.../apache/tez/dag/api/TezConfiguration.java | 19 ++++++++++++++
.../org/apache/tez/common/TestTezUtils.java | 26 ++++++++++++++++++++
.../org/apache/tez/dag/app/DAGAppMaster.java | 8 ++++++
.../org/apache/tez/runtime/task/TezChild.java | 9 +++++++
6 files changed, 81 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b6b04de4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3712093..20b2870 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3326. Display JVM system properties in AM and task logs.
TEZ-3009. Errors that occur during container task acquisition are not logged.
TEZ-2852. TestVertexImpl fails due to race in AsyncDispatcher.
TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion
for AppLaunchedEvent.
http://git-wip-us.apache.org/repos/asf/tez/blob/b6b04de4/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 93d373b..3aa1914 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -20,6 +20,7 @@ package org.apache.tez.common;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.records.DAGProtos;
@@ -66,6 +68,22 @@ public class TezUtils {
TezClientUtils.addLog4jSystemProperties(logLevel, vargs);
}
+ public static String getSystemPropertiesToLog(Configuration conf) {
+ Collection <String> keys = conf.getTrimmedStringCollection(
+ TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG);
+ if (keys.isEmpty()) {
+ keys = TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT;
+ }
+ StringBuilder sb = new StringBuilder();
+
sb.append("\n/************************************************************\n");
+ sb.append("[system properties]\n");
+ for (String key : keys) {
+ sb.append(key).append(": ").append(System.getProperty(key)).append('\n');
+ }
+ sb.append("************************************************************/");
+ return sb.toString();
+ }
+
/**
* Convert a Configuration to compressed ByteString using Protocol buffer
http://git-wip-us.apache.org/repos/asf/tez/blob/b6b04de4/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 15e937f..5aa8f7e 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -19,7 +19,11 @@
package org.apache.tez.dag.api;
import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -1647,4 +1651,19 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES =
TEZ_PREFIX + "am.ats.v15.override.summary-types";
public static final boolean TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT =
true;
+
+ /**
+ * String value. Determines what JVM properties will be logged for debugging
purposes
+ * in the AM and Task runtime logs.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG =
+ TEZ_PREFIX + "tez.jvm.system-properties-to-log";
+ public static final List<String> TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT =
+ Collections.unmodifiableList(Arrays.asList(
+ "os.name","os.version","java.home","java.runtime.version",
+ "java.vendor","java.version","java.vm.name","java.class.path",
+ "java.io.tmpdir","user.dir","user.name"));
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b6b04de4/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
index c88fa67..2eab776 100644
--- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
+++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -17,11 +17,16 @@
package org.apache.tez.common;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.util.BitSet;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.UserPayload;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -225,5 +230,26 @@ public class TestTezUtils {
}
+ @Test
+ public void testLogSystemProperties() throws Exception {
+ Configuration conf = new Configuration();
+ // test default logging
+ conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, " ");
+ String value = TezUtils.getSystemPropertiesToLog(conf);
+ for(String key: TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT)
{
+ assertTrue(value.contains(key));
+ }
+
+ // test logging of selected keys
+ String classpath = "java.class.path";
+ String os = "os.name";
+ String version = "java.version";
+ conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, classpath + ",
" + os);
+ value = TezUtils.getSystemPropertiesToLog(conf);
+ assertNotNull(value);
+ assertTrue(value.contains(classpath));
+ assertTrue(value.contains(os));
+ assertFalse(value.contains(version));
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b6b04de4/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 7ca7118..cc07fb7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -2356,6 +2356,14 @@ public class DAGAppMaster extends AbstractService {
ShutdownHookManager.get().addShutdownHook(
new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
+ // log the system properties
+ if (LOG.isInfoEnabled()) {
+ String systemPropsToLog = TezUtils.getSystemPropertiesToLog(conf);
+ if (systemPropsToLog != null) {
+ LOG.info(systemPropsToLog);
+ }
+ }
+
initAndStartAppMaster(appMaster, conf);
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/tez/blob/b6b04de4/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git
a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 3c2d19a..2255ed7 100644
---
a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -54,6 +54,7 @@ import org.apache.tez.common.ContainerTask;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezLocalResource;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.TezUtils;
import org.apache.tez.common.TezUtilsInternal;
import org.apache.tez.common.counters.Limits;
import org.apache.tez.common.security.JobTokenIdentifier;
@@ -502,6 +503,14 @@ public class TezChild {
HadoopShim hadoopShim = new HadoopShimsLoader(defaultConf).getHadoopShim();
+ // log the system properties
+ if (LOG.isInfoEnabled()) {
+ String systemPropsToLog = TezUtils.getSystemPropertiesToLog(defaultConf);
+ if (systemPropsToLog != null) {
+ LOG.info(systemPropsToLog);
+ }
+ }
+
TezChild tezChild = newTezChild(defaultConf, host, port,
containerIdentifier,
tokenIdentifier, attemptNumber, localDirs,
System.getenv(Environment.PWD.name()),
System.getenv(), pid, new
ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())),
