TEZ-2698. rebase 08/05 (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/73ce6c57 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/73ce6c57 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/73ce6c57 Branch: refs/heads/TEZ-2003 Commit: 73ce6c578188abd6d9c19b210c1f94e821b7fabe Parents: 267fe73 Author: Siddharth Seth <[email protected]> Authored: Thu Aug 6 02:20:32 2015 -0700 Committer: Siddharth Seth <[email protected]> Committed: Fri Aug 14 13:47:10 2015 -0700 ---------------------------------------------------------------------- TEZ-2003-CHANGES.txt | 1 + .../test/java/org/apache/tez/client/TestTezClientUtils.java | 2 +- .../org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java | 1 + .../java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java | 5 ++++- .../src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java | 6 +++--- .../apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java | 4 +--- .../apache/tez/runtime/library/common/shuffle/TestFetcher.java | 2 +- 7 files changed, 12 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/73ce6c57/TEZ-2003-CHANGES.txt ---------------------------------------------------------------------- diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt index f921739..b133ea3 100644 --- a/TEZ-2003-CHANGES.txt +++ b/TEZ-2003-CHANGES.txt @@ -43,5 +43,6 @@ ALL CHANGES: TEZ-2657. Add tests for client side changes - specifying plugins, etc. TEZ-2626. Fix log lines with DEBUG in messages, consolidate TEZ-2003 TODOs. TEZ-2126. Add unit tests for verifying multiple schedulers, launchers, communicators. + TEZ-2698. rebase 08/05 INCOMPATIBLE CHANGES: http://git-wip-us.apache.org/repos/asf/tez/blob/73ce6c57/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index 8f40bbd..d1033b2 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -225,7 +225,7 @@ public class TestTezClientUtils { appId, null, "dagname", amConf, m, credentials, false, - new TezApiVersionInfo(), null); + new TezApiVersionInfo(), null, null); assertEquals(testpriority, appcontext.getPriority().getPriority()); } http://git-wip-us.apache.org/repos/asf/tez/blob/73ce6c57/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java index 7b97738..6c1dad9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java @@ -32,6 +32,7 @@ import org.apache.commons.collections4.ListUtils; import org.apache.tez.dag.api.NamedEntityDescriptor; import org.apache.tez.dag.api.TezConstants; import org.apache.tez.dag.api.UserPayload; +import org.apache.tez.dag.app.dag.event.TaskAttemptEventType; import org.apache.tez.serviceplugins.api.ContainerEndReason; import org.apache.tez.dag.app.dag.event.TaskAttemptEvent; import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate; http://git-wip-us.apache.org/repos/asf/tez/blob/73ce6c57/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java index fb6d5e7..d3f1c44 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java @@ -163,7 +163,10 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator { } server.start(); - this.address = NetUtils.getConnectAddress(server); + InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server); + this.address = NetUtils.createSocketAddrForHost( + serverBindAddress.getAddress().getCanonicalHostName(), + serverBindAddress.getPort()); LOG.info("Instantiated TezTaskCommunicator RPC at " + this.address); } catch (IOException e) { throw new TezUncheckedException(e); http://git-wip-us.apache.org/repos/asf/tez/blob/73ce6c57/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java index 2e6e568..b04b461 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java @@ -401,7 +401,7 @@ public class MockDAGAppMaster extends DAGAppMaster { List<TezEvent> events = Lists.newArrayListWithCapacity( cData.taskSpec.getOutputs().size() + 1); if (cData.numUpdates == 0 && eventsDelegate != null) { - eventsDelegate.getEvents(cData.taskSpec, events, getContext().getClock().getTime()); + eventsDelegate.getEvents(cData.taskSpec, events, MockDAGAppMaster.this.getContext().getClock().getTime()); } TezCounters counters = null; if (countersDelegate != null) { @@ -416,7 +416,7 @@ public class MockDAGAppMaster extends DAGAppMaster { float progress = updateProgress ? cData.numUpdates/maxUpdates : 0f; events.add(new TezEvent(new TaskStatusUpdateEvent(counters, progress, stats), new EventMetaData( EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId), - getContext().getClock().getTime())); + MockDAGAppMaster.this.getContext().getClock().getTime())); // TezHeartbeatRequest request = new TezHeartbeatRequest(cData.numUpdates, events, // cData.cIdStr, cData.taId, cData.nextFromEventId, 50000); TaskHeartbeatRequest request = @@ -431,7 +431,7 @@ public class MockDAGAppMaster extends DAGAppMaster { List<TezEvent> events = Collections.singletonList(new TezEvent( new TaskAttemptCompletedEvent(), new EventMetaData( EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId), - getContext().getClock().getTime())); + MockDAGAppMaster.this.getContext().getClock().getTime())); TaskHeartbeatRequest request = new TaskHeartbeatRequest(cData.cIdStr, cData.taId, events, cData.nextFromEventId, cData.nextPreRoutedFromEventId, 10000); http://git-wip-us.apache.org/repos/asf/tez/blob/73ce6c57/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java index 639c487..4d404b9 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java @@ -89,7 +89,7 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; @SuppressWarnings("unchecked") -// TODO TEZ-2003 Rename to TestTezTaskCommunicator +// TODO TEZ-2003 (post) TEZ-2696 Rename to TestTezTaskCommunicator public class TestTaskAttemptListenerImplTezDag { private ApplicationId appId; private ApplicationAttemptId appAttemptId; @@ -153,8 +153,6 @@ public class TestTaskAttemptListenerImplTezDag { Lists.newArrayList( new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null) .setUserPayload(defaultPayload))); - TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(); - TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical(); taskSpec = mock(TaskSpec.class); doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID(); http://git-wip-us.apache.org/repos/asf/tez/blob/73ce6c57/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java index 08efb3e..42b957b 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java @@ -273,7 +273,7 @@ public class TestFetcher { int partition = 42; FetcherCallback callback = mock(FetcherCallback.class); Fetcher.FetcherBuilder builder = new Fetcher.FetcherBuilder(callback, null, null, - ApplicationId.newInstance(0, 1), null, "fetcherTest", conf, true, HOST, PORT, false); + ApplicationId.newInstance(0, 1), 1, null, "fetcherTest", conf, true, HOST, PORT, false); builder.assignWork(HOST, PORT, partition, Arrays.asList(srcAttempts)); Fetcher fetcher = spy(builder.build()); fetcher.populateRemainingMap(new LinkedList<InputAttemptIdentifier>(Arrays.asList(srcAttempts)));
