Repository: tez Updated Branches: refs/heads/master 8a1ac4458 -> 327e02a76
TEZ-2874. Improved logging for caller context. (hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/327e02a7 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/327e02a7 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/327e02a7 Branch: refs/heads/master Commit: 327e02a761eb01c74acdf5b7af65f79e23b4e28b Parents: 8a1ac44 Author: Hitesh Shah <[email protected]> Authored: Fri Oct 9 19:35:05 2015 -0700 Committer: Hitesh Shah <[email protected]> Committed: Fri Oct 9 19:36:02 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../java/org/apache/tez/client/CallerContext.java | 13 +++++++++++-- .../main/java/org/apache/tez/client/TezClient.java | 17 +++++++++++++---- .../src/main/java/org/apache/tez/dag/api/DAG.java | 5 +++++ .../test/java/org/apache/tez/dag/api/TestDAG.java | 15 ++++++++------- .../java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +- 6 files changed, 40 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 1ced5d8..32aca94 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ Release 0.8.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2874. Improved logging for caller context. TEZ-2875. Enable missing tests in TestAnalyzer TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails TEZ-1788. Allow vertex level disabling of speculation @@ -202,6 +203,7 @@ Release 0.7.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES + TEZ-2874. Improved logging for caller context. TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails TEZ-2868. Fix setting Caller Context in Tez Examples. TEZ-2860. NPE in DAGClientImpl. http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-api/src/main/java/org/apache/tez/client/CallerContext.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/CallerContext.java b/tez-api/src/main/java/org/apache/tez/client/CallerContext.java index ba68851..809cf47 100644 --- a/tez-api/src/main/java/org/apache/tez/client/CallerContext.java +++ b/tez-api/src/main/java/org/apache/tez/client/CallerContext.java @@ -162,10 +162,19 @@ public class CallerContext { @Override public String toString() { - return "context=" + context + return "{ context=" + context + ", callerType=" + callerType + ", callerId=" + callerId - + ", blob=" + blob; + + ", blob=" + blob + + " }"; } + public String contextAsSimpleString() { + return "{ context=" + context + + ", callerType=" + callerType + + ", callerId=" + callerId + + " }"; + } + + } http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-api/src/main/java/org/apache/tez/client/TezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index 0c50d86..ff131e0 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -57,7 +57,6 @@ import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezConstants; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.TezReflectionException; -import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB; import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAMStatusRequestProto; @@ -464,10 +463,15 @@ public class TezClient { verifySessionStateForSubmission(); String dagId = null; + String callerContextStr = ""; + if (dag.getCallerContext() != null) { + callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString(); + } LOG.info("Submitting dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId - + ", dagName=" + dag.getName()); + + ", dagName=" + dag.getName() + + callerContextStr); if (!additionalLocalResources.isEmpty()) { for (LocalResource lr : additionalLocalResources.values()) { @@ -868,13 +872,18 @@ public class TezClient { // Add credentials for tez-local resources. Map<String, LocalResource> tezJarResources = getTezJarResources(credentials); ApplicationSubmissionContext appContext = TezClientUtils - .createApplicationSubmissionContext( + .createApplicationSubmissionContext( appId, dag, dag.getName(), amConfig, tezJarResources, credentials, usingTezArchiveDeploy, apiVersionInfo, historyACLPolicyManager, servicePluginsDescriptor, javaOptsChecker); + String callerContextStr = ""; + if (dag.getCallerContext() != null) { + callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString(); + } LOG.info("Submitting DAG to YARN" + ", applicationId=" + appId - + ", dagName=" + dag.getName()); + + ", dagName=" + dag.getName() + + callerContextStr); frameworkClient.submitApplication(appContext); ApplicationReport appReport = frameworkClient.getApplicationReport(appId); http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java index e4fdc18..c02d86f 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java @@ -1084,4 +1084,9 @@ public class DAG { } return false; } + + public synchronized CallerContext getCallerContext() { + return this.callerContext; + } + } http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java index 24c20b5..839e780 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java @@ -311,24 +311,25 @@ public class TestDAG { @Test public void testCallerContext() { - DAG dag = DAG.create("dag1"); try { - CallerContext callerContext = CallerContext.create("ctxt", "", "", "desc"); + CallerContext.create("ctxt", "", "", "desc"); Assert.fail("Expected failure for invalid args"); } catch (Exception e) { // Expected } try { - CallerContext callerContext = CallerContext.create("", "desc"); + CallerContext.create("", "desc"); Assert.fail("Expected failure for invalid args"); } catch (Exception e) { // Expected } - CallerContext callerContext; - callerContext = CallerContext.create("ctxt", "a", "a", "desc"); - callerContext = CallerContext.create("ctxt", "desc"); - callerContext = CallerContext.create("ctxt", null); + CallerContext.create("ctxt", "a", "a", "desc"); + CallerContext.create("ctxt", null); + + CallerContext callerContext = CallerContext.create("ctxt", "desc"); + Assert.assertTrue(callerContext.toString().contains("desc")); + Assert.assertFalse(callerContext.contextAsSimpleString().contains("desc")); } http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index e165397..d774b9a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2308,7 +2308,7 @@ public class DAGAppMaster extends AbstractService { if (dagPlan.hasCallerContext()) { CallerContext callerContext = DagTypeConverters.convertCallerContextFromProto( dagPlan.getCallerContext()); - callerContextStr = ", callerContext=" + callerContext.toString(); + callerContextStr = ", callerContext=" + callerContext.contextAsSimpleString(); } LOG.info("Running DAG: " + dagPlan.getName() + callerContextStr);
