Repository: tez
Updated Branches:
refs/heads/branch-0.7 7e15ba5aa -> 55df029a8
TEZ-3326. Display JVM system properties in AM and task logs. (Eric Badger via
hitesh)
(cherry picked from commit ae984705e01b357ba5b7fc972b58be4d75f5016d)
Conflicts:
CHANGES.txt
(cherry picked from commit b6b04de4627f096f6b182cb44185158efdb6d561)
Conflicts:
tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/55df029a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/55df029a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/55df029a
Branch: refs/heads/branch-0.7
Commit: 55df029a87004f9ef8a9d4ca223fa016b8353387
Parents: 7e15ba5
Author: Hitesh Shah <[email protected]>
Authored: Thu Sep 1 14:13:49 2016 -0700
Committer: Hitesh Shah <[email protected]>
Committed: Thu Sep 1 14:27:57 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/common/TezUtils.java | 18 ++++++++++++++
.../apache/tez/dag/api/TezConfiguration.java | 17 +++++++++++++
.../org/apache/tez/common/TestTezUtils.java | 26 ++++++++++++++++++++
.../org/apache/tez/dag/app/DAGAppMaster.java | 9 +++++++
.../org/apache/tez/runtime/task/TezChild.java | 10 ++++++++
6 files changed, 81 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1aeed7d..3bba3e4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3326. Display JVM system properties in AM and task logs.
TEZ-3009. Errors that occur during container task acquisition are not logged.
TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion
for AppLaunchedEvent.
TEZ-3286. Allow clients to set processor reserved memory per vertex (instead
of per container).
http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 93d373b..3aa1914 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -20,6 +20,7 @@ package org.apache.tez.common;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.records.DAGProtos;
@@ -66,6 +68,22 @@ public class TezUtils {
TezClientUtils.addLog4jSystemProperties(logLevel, vargs);
}
+ public static String getSystemPropertiesToLog(Configuration conf) {
+ Collection <String> keys = conf.getTrimmedStringCollection(
+ TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG);
+ if (keys.isEmpty()) {
+ keys = TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT;
+ }
+ StringBuilder sb = new StringBuilder();
+
sb.append("\n/************************************************************\n");
+ sb.append("[system properties]\n");
+ for (String key : keys) {
+ sb.append(key).append(": ").append(System.getProperty(key)).append('\n');
+ }
+ sb.append("************************************************************/");
+ return sb.toString();
+ }
+
/**
* Convert a Configuration to compressed ByteString using Protocol buffer
http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index a96d560..9b32b5f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -19,7 +19,11 @@
package org.apache.tez.dag.api;
import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -1450,4 +1454,17 @@ public class TezConfiguration extends Configuration {
TEZ_PREFIX + "java.opts.checker.enabled";
public static final boolean TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT =
true;
+ /**
+ * String value. Determines what JVM properties will be logged for debugging
purposes
+ * in the AM and Task runtime logs.
+ */
+ @ConfigurationScope(Scope.AM)
+ public static final String TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG =
+ TEZ_PREFIX + "tez.jvm.system-properties-to-log";
+ public static final List<String> TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT =
+ Collections.unmodifiableList(Arrays.asList(
+ "os.name","os.version","java.home","java.runtime.version",
+ "java.vendor","java.version","java.vm.name","java.class.path",
+ "java.io.tmpdir","user.dir","user.name"));
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
index c88fa67..2eab776 100644
--- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
+++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -17,11 +17,16 @@
package org.apache.tez.common;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.util.BitSet;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.UserPayload;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -225,5 +230,26 @@ public class TestTezUtils {
}
+ @Test
+ public void testLogSystemProperties() throws Exception {
+ Configuration conf = new Configuration();
+ // test default logging
+ conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, " ");
+ String value = TezUtils.getSystemPropertiesToLog(conf);
+ for(String key: TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT)
{
+ assertTrue(value.contains(key));
+ }
+
+ // test logging of selected keys
+ String classpath = "java.class.path";
+ String os = "os.name";
+ String version = "java.version";
+ conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, classpath + ",
" + os);
+ value = TezUtils.getSystemPropertiesToLog(conf);
+ assertNotNull(value);
+ assertTrue(value.contains(classpath));
+ assertTrue(value.contains(os));
+ assertFalse(value.contains(version));
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 2b33686..673f2fe 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -58,6 +58,7 @@ import java.util.regex.Pattern;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
+import org.apache.tez.common.TezUtils;
import org.apache.tez.client.CallerContext;
import org.apache.tez.dag.api.SessionNotRunning;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEventDagCleanup;
@@ -2178,6 +2179,14 @@ public class DAGAppMaster extends AbstractService {
ShutdownHookManager.get().addShutdownHook(
new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
+ // log the system properties
+ if (LOG.isInfoEnabled()) {
+ String systemPropsToLog = TezUtils.getSystemPropertiesToLog(conf);
+ if (systemPropsToLog != null) {
+ LOG.info(systemPropsToLog);
+ }
+ }
+
initAndStartAppMaster(appMaster, conf);
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git
a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 4f00998..757b886 100644
---
a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -55,6 +55,7 @@ import org.apache.tez.common.ContainerTask;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezLocalResource;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.TezUtils;
import org.apache.tez.common.TezUtilsInternal;
import org.apache.tez.common.counters.Limits;
import org.apache.tez.common.security.JobTokenIdentifier;
@@ -488,6 +489,15 @@ public class TezChild {
TezUtilsInternal.addUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()),
defaultConf);
UserGroupInformation.setConfiguration(defaultConf);
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
+
+ // log the system properties
+ if (LOG.isInfoEnabled()) {
+ String systemPropsToLog = TezUtils.getSystemPropertiesToLog(defaultConf);
+ if (systemPropsToLog != null) {
+ LOG.info(systemPropsToLog);
+ }
+ }
+
TezChild tezChild = newTezChild(defaultConf, host, port,
containerIdentifier,
tokenIdentifier, attemptNumber, localDirs,
System.getenv(Environment.PWD.name()),
System.getenv(), pid, new
ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())),
