TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5150c203 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5150c203 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5150c203 Branch: refs/heads/TEZ-2003 Commit: 5150c2034e9833095936bc1972a4d98d009b71a3 Parents: 8b79df5 Author: Siddharth Seth <[email protected]> Authored: Mon Feb 23 20:53:24 2015 -0800 Committer: Siddharth Seth <[email protected]> Committed: Thu Aug 6 01:25:09 2015 -0700 ---------------------------------------------------------------------- TEZ-2003-CHANGES.txt | 1 + .../src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +- .../java/org/apache/tez/dag/app/dag/impl/VertexImpl.java | 4 ++++ .../org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java | 1 + .../dag/app/launcher/TezTestServiceContainerLauncher.java | 8 +++++--- 5 files changed, 12 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/5150c203/TEZ-2003-CHANGES.txt ---------------------------------------------------------------------- diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt index 4b0a12b..4377f57 100644 --- a/TEZ-2003-CHANGES.txt +++ b/TEZ-2003-CHANGES.txt @@ -7,5 +7,6 @@ ALL CHANGES: TEZ-2123. Fix component managers to use pluggable components. (Enable hybrid mode) TEZ-2125. Create a task communicator for local mode. Allow tasks to run in the AM. TEZ-2131. Add additional tests for tasks running in the AM. + TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers. INCOMPATIBLE CHANGES: http://git-wip-us.apache.org/repos/asf/tez/blob/5150c203/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 43f8794..6ab0f8e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -1547,7 +1547,7 @@ public class DAGAppMaster extends AbstractService { @Override public Integer getContainerLauncherIdentifier(String name) { - return taskCommunicators.get(name); + return containerLaunchers.get(name); } @Override http://git-wip-us.apache.org/repos/asf/tez/blob/5150c203/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index 7733ef9..bdab984 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -1003,6 +1003,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl .get(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME, tezDefaultComponentName); String containerLauncherName = vertexConf .get(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME, tezDefaultComponentName); + + LOG.info("Vertex: " + logIdentifier + " configured with TaskScheduler=" + taskSchedulerName + + ", ContainerLauncher=" + containerLauncherName + ", TaskComm=" + taskCommName); + taskSchedulerIdentifier = appContext.getTaskScheduerIdentifier(taskSchedulerName); taskCommunicatorIdentifier = appContext.getTaskCommunicatorIdentifier(taskCommName); containerLauncherIdentifier = appContext.getContainerLauncherIdentifier(containerLauncherName); http://git-wip-us.apache.org/repos/asf/tez/blob/5150c203/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java index ba46a67..9a8bf16 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java @@ -174,6 +174,7 @@ public class TaskSchedulerEventHandler extends AbstractService for (int i = 0 ; i < schedulerClasses.length ; i++) { // Copy over the rest. this.taskSchedulerClasses[i] = schedulerClasses[i]; } + this.taskSchedulerClasses[foundYarnTaskSchedulerIndex] = TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT; } else { this.taskSchedulerClasses = schedulerClasses; } http://git-wip-us.apache.org/repos/asf/tez/blob/5150c203/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java ---------------------------------------------------------------------- diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java index 27356bc..cb6308c 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.util.Clock; import org.apache.tez.dag.app.AppContext; @@ -54,6 +55,7 @@ public class TezTestServiceContainerLauncher extends AbstractService implements private final int servicePort; private final TezTestServiceCommunicator communicator; private final Clock clock; + private final ApplicationAttemptId appAttemptId; // Configuration passed in here to set up final parameters @@ -70,6 +72,7 @@ public class TezTestServiceContainerLauncher extends AbstractService implements this.communicator = new TezTestServiceCommunicator(numThreads); this.context = appContext; this.tokenIdentifier = context.getApplicationID().toString(); + this.appAttemptId = appContext.getApplicationAttemptId(); this.tal = tal; } @@ -128,9 +131,8 @@ public class TezTestServiceContainerLauncher extends AbstractService implements RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder(); InetSocketAddress address = tal.getTaskCommunicator(event.getTaskCommId()).getAddress(); builder.setAmHost(address.getHostName()).setAmPort(address.getPort()); - builder.setAppAttemptNumber(event.getContainer().getId().getApplicationAttemptId().getAttemptId()); - builder.setApplicationIdString( - event.getContainer().getId().getApplicationAttemptId().getApplicationId().toString()); + builder.setAppAttemptNumber(appAttemptId.getAttemptId()); + builder.setApplicationIdString(appAttemptId.getApplicationId().toString()); builder.setTokenIdentifier(tokenIdentifier); builder.setContainerIdString(event.getContainer().getId().toString()); builder.setCredentialsBinary(
