Repository: tez Updated Branches: refs/heads/branch-0.8 9a4e7eabc -> ec8e6e837
TEZ-3550. Provide access to sessionId/dagId via DagClient. (sseth) (cherry picked from commit cd44c906f762c24a13ddabe691e55c879c223eac) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ec8e6e83 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ec8e6e83 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ec8e6e83 Branch: refs/heads/branch-0.8 Commit: ec8e6e837bab952f43986c860629af5d5919293b Parents: 9a4e7ea Author: Siddharth Seth <[email protected]> Authored: Tue Feb 14 10:46:18 2017 -0800 Committer: Siddharth Seth <[email protected]> Committed: Tue Feb 14 10:47:19 2017 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../java/org/apache/tez/dag/api/client/DAGClient.java | 13 +++++++++++++ .../org/apache/tez/dag/api/client/DAGClientImpl.java | 10 ++++++++++ .../apache/tez/dag/api/client/DAGClientInternal.java | 14 +++++++++++++- .../tez/dag/api/client/DAGClientTimelineImpl.java | 11 ++++++++++- .../tez/dag/api/client/rpc/DAGClientRPCImpl.java | 10 +++++++++- .../java/org/apache/tez/client/TestTezClient.java | 5 ++++- .../apache/tez/dag/api/client/rpc/TestDAGClient.java | 2 ++ .../org/apache/tez/dag/api/client/MRDAGClient.java | 10 ++++++++++ 9 files changed, 72 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7a401b2..7d886c1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3550. Provide access to sessionId/dagId via DagClient. TEZ-3609. Improve ATSv15 performance for DAG entities read calls. TEZ-3244. Allow overlap of input and output memory when they are not concurrent TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java index 9b11b96..c70da75 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java @@ -84,6 +84,19 @@ public abstract class DAGClient implements Closeable { throws IOException, TezException; /** + * Get the dag identifier for the currently executing dag. This is a string + * which represents this dag + * @return the dag identifier + */ + public abstract String getDagIdentifierString(); + + /** + * Get the session identifier for the session in which this dag is running + * @return the session identifier + */ + public abstract String getSessionIdentifierString(); + + /** * Kill a running DAG * */ http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java index af67ee8..8a277ed 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java @@ -318,6 +318,16 @@ public class DAGClientImpl extends DAGClient { } @Override + public String getDagIdentifierString() { + return realClient.getDagIdentifierString(); + } + + @Override + public String getSessionIdentifierString() { + return realClient.getSessionIdentifierString(); + } + + @Override public void tryKillDAG() throws IOException, TezException { if (!dagCompleted) { realClient.tryKillDAG(); http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java index bb236a3..a3c898a 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java @@ -25,7 +25,6 @@ import java.util.Set; import javax.annotation.Nullable; import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.tez.dag.api.TezException; @@ -85,6 +84,19 @@ public abstract class DAGClientInternal implements Closeable { throws IOException, TezException, ApplicationNotFoundException; /** + * Get the dag identifier for the currently executing dag. This is a string + * which represents this dag + * @return the dag identifier + */ + public abstract String getDagIdentifierString(); + + /** + * Get the session identifier for the session in which this dag is running + * @return the session identifier + */ + public abstract String getSessionIdentifierString(); + + /** * Kill a running DAG * */ http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java index ffd91b7..d34dbf0 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java @@ -38,7 +38,6 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -182,6 +181,16 @@ public class DAGClientTimelineImpl extends DAGClientInternal { } @Override + public String getDagIdentifierString() { + return dagId; + } + + @Override + public String getSessionIdentifierString() { + return appId.toString(); + } + + @Override public void tryKillDAG() throws IOException, TezException { throw new TezException("tryKillDAG is unsupported for DAGClientTimelineImpl"); } http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java index ff48755..d8ace91 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java @@ -41,7 +41,6 @@ import org.apache.tez.dag.api.DAGNotRunningException; import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; -import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.DagStatusSource; import org.apache.tez.dag.api.client.StatusGetOpts; @@ -130,6 +129,15 @@ public class DAGClientRPCImpl extends DAGClientInternal { return null; } + @Override + public String getDagIdentifierString() { + return dagId.toString(); + } + + @Override + public String getSessionIdentifierString() { + return appId.toString(); + } @Override http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index a896de3..924ce61 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -280,7 +280,8 @@ public class TestTezClient { DAGClient dagClient = client.submitDAG(dag); assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); - + assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString()); + if (isSession) { verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); verify(client.sessionAmProxy, times(1)).submitDAG((RpcController)any(), (SubmitDAGRequestProto) any()); @@ -319,6 +320,7 @@ public class TestTezClient { // same app master verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); + assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString()); // additional resource is sent ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class); verify(client.sessionAmProxy, times(2)).submitDAG((RpcController)any(), captor1.capture()); @@ -328,6 +330,7 @@ public class TestTezClient { } else { // new app master assertTrue(dagClient.getExecutionContext().contains(appId2.toString())); + assertEquals(dagClient.getSessionIdentifierString(), appId2.toString()); verify(client.mockYarnClient, times(2)).submitApplication(captor.capture()); // additional resource is added ApplicationSubmissionContext context = captor.getValue(); http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java index 674781e..902f32b 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java @@ -210,6 +210,8 @@ public class TestDAGClient { @Test(timeout = 5000) public void testApp() throws IOException, TezException, ServiceException{ assertTrue(dagClient.getExecutionContext().contains(mockAppId.toString())); + assertEquals(mockAppId.toString(), dagClient.getSessionIdentifierString()); + assertEquals(dagIdStr, dagClient.getDagIdentifierString()); DAGClientRPCImpl realClient = (DAGClientRPCImpl)((DAGClientImpl)dagClient).getRealClient(); assertEquals(mockAppReport, realClient.getApplicationReportInternal()); } http://git-wip-us.apache.org/repos/asf/tez/blob/ec8e6e83/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java b/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java index d743feb..42b52e0 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java @@ -66,6 +66,16 @@ public class MRDAGClient extends DAGClient { } @Override + public String getDagIdentifierString() { + return realClient.getDagIdentifierString(); + } + + @Override + public String getSessionIdentifierString() { + return realClient.getSessionIdentifierString(); + } + + @Override public void tryKillDAG() throws IOException, TezException { realClient.tryKillDAG(); }
