TEZ-3155. Addendum patch for logging. Support a way to submit DAGs to a session where the DAG plan exceeds hadoop ipc limits. (Zhiyuan Yang via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a8c93048 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a8c93048 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a8c93048 Branch: refs/heads/master Commit: a8c9304815e447945d226c40690a6d69304d13e4 Parents: fac2f5f Author: Hitesh Shah <[email protected]> Authored: Mon Mar 14 11:18:22 2016 -0700 Committer: Hitesh Shah <[email protected]> Committed: Mon Mar 14 11:18:22 2016 -0700 ---------------------------------------------------------------------- tez-api/src/main/java/org/apache/tez/client/TezClient.java | 5 +++++ 1 file changed, 5 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/a8c93048/tez-api/src/main/java/org/apache/tez/client/TezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index be59e2f..a6a256c 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -541,6 +541,11 @@ public class TezClient { serializedSubmitDAGPlanRequestCounter.incrementAndGet()); try (FSDataOutputStream fsDataOutputStream = stagingFs.create(dagPlanPath, false)) { + LOG.info("Send dag plan using YARN local resources since it's too large" + + ", dag plan size=" + request.getSerializedSize() + + ", max dag plan size through IPC=" + maxSubmitDAGRequestSizeThroughIPC + + ", max IPC message size= " + amConfig.getTezConfiguration().getInt( + CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT)); request.writeTo(fsDataOutputStream); request = requestBuilder.clear().setSerializedRequestPath(stagingFs.resolvePath(dagPlanPath).toString()).build(); }
