Repository: hadoop Updated Branches: refs/heads/branch-2 c617dba49 -> 4d69741a6
YARN-7974. Allow updating application tracking url after registration. (Jonathan Hung via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d69741a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d69741a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d69741a Branch: refs/heads/branch-2 Commit: 4d69741a61b8d29774a1e5f5075406808fc0594a Parents: c617dba Author: Arun Suresh <asur...@apache.org> Authored: Wed Sep 26 00:08:10 2018 -0700 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Sep 26 00:08:10 2018 -0700 ---------------------------------------------------------------------- .../api/protocolrecords/AllocateRequest.java | 47 +++++++++++- .../src/main/proto/yarn_service_protos.proto | 1 + .../hadoop/yarn/client/api/AMRMClient.java | 11 +++ .../yarn/client/api/async/AMRMClientAsync.java | 11 +++ .../api/async/impl/AMRMClientAsyncImpl.java | 5 ++ .../yarn/client/api/impl/AMRMClientImpl.java | 11 +++ .../yarn/client/api/impl/TestAMRMClient.java | 77 ++++++++++++++++++++ .../impl/pb/AllocateRequestPBImpl.java | 27 ++++++- .../resourcemanager/DefaultAMSProcessor.java | 2 +- .../rmapp/attempt/RMAppAttemptImpl.java | 22 ++++++ .../event/RMAppAttemptStatusupdateEvent.java | 11 +++ .../TestApplicationMasterService.java | 34 +++++++++ .../server/resourcemanager/TestRMRestart.java | 45 ++++++++++++ 13 files changed, 300 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index da68d8c..8f6a204 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -100,7 +100,21 @@ public abstract class AllocateRequest { allocateRequest.setIncreaseRequests(increaseRequests); return allocateRequest; } - + + @Public + @Unstable + public static AllocateRequest newInstance(int responseID, float appProgress, + List<ResourceRequest> resourceAsk, + List<ContainerId> containersToBeReleased, + ResourceBlacklistRequest resourceBlacklistRequest, + String trackingUrl) { + return AllocateRequest.newBuilder().responseId(responseID) + .progress(appProgress).askList(resourceAsk) + .releaseList(containersToBeReleased) + .resourceBlacklistRequest(resourceBlacklistRequest) + .trackingUrl(trackingUrl).build(); + } + @Public @Unstable public static AllocateRequest newInstance(int responseID, float appProgress, @@ -256,6 +270,22 @@ public abstract class AllocateRequest { public abstract void setUpdateRequests( List<UpdateContainerRequest> updateRequests); + /** + * Get the tracking url update for this heartbeat. + * @return tracking url to update this application with + */ + @Public + @Unstable + public abstract String getTrackingUrl(); + + /** + * Set the new tracking url for this application. + * @param trackingUrl the new tracking url + */ + @Public + @Unstable + public abstract void setTrackingUrl(String trackingUrl); + @Public @Unstable public static AllocateRequestBuilder newBuilder() { @@ -358,6 +388,19 @@ public abstract class AllocateRequest { } /** + * Set the <code>trackingUrl</code> of the request. + * @see AllocateRequest#setTrackingUrl(String) + * @param trackingUrl new tracking url + * @return {@link AllocateRequestBuilder} + */ + @Public + @Unstable + public AllocateRequestBuilder trackingUrl(String trackingUrl) { + allocateRequest.setTrackingUrl(trackingUrl); + return this; + } + + /** * Return generated {@link AllocateRequest} object. * @return {@link AllocateRequest} */ @@ -367,4 +410,4 @@ public abstract class AllocateRequest { return allocateRequest; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 8301971..a5fc568 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -89,6 +89,7 @@ message AllocateRequestProto { optional float progress = 5; repeated ContainerResourceIncreaseRequestProto increase_request = 6; repeated UpdateContainerRequestProto update_requests = 7; + optional string tracking_url = 11; } message NMTokenProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java index b8eb58c..989a7e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java @@ -701,6 +701,17 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends } /** + * Update application's tracking url on next heartbeat. + * + * @param trackingUrl new tracking url for this application + */ + @Public + @InterfaceStability.Unstable + public void updateTrackingUrl(String trackingUrl) { + // Unimplemented. + } + + /** * Wait for <code>check</code> to return true for each 1000 ms. * See also {@link #waitFor(com.google.common.base.Supplier, int)} * and {@link #waitFor(com.google.common.base.Supplier, int, int)} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java index 1ecfe1f..54d7cfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java @@ -374,6 +374,17 @@ extends AbstractService { List<String> blacklistRemovals); /** + * Update application's tracking url on next heartbeat. + * + * @param trackingUrl new tracking url for this application + */ + @Public + @Unstable + public void updateTrackingUrl(String trackingUrl) { + // Unimplemented. + } + + /** * Wait for <code>check</code> to return true for each 1000 ms. * See also {@link #waitFor(com.google.common.base.Supplier, int)} * and {@link #waitFor(com.google.common.base.Supplier, int, int)} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java index 0513211..0a01514 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java @@ -250,6 +250,11 @@ extends AMRMClientAsync<T> { List<String> blacklistRemovals) { client.updateBlacklist(blacklistAdditions, blacklistRemovals); } + + @Override + public void updateTrackingUrl(String trackingUrl) { + client.updateTrackingUrl(trackingUrl); + } private class HeartbeatThread extends Thread { public HeartbeatThread() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java index e4561db..8217e34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java @@ -94,6 +94,7 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> { protected String appHostName; protected int appHostPort; protected String appTrackingUrl; + protected String newTrackingUrl; protected ApplicationMasterProtocol rmClient; protected Resource clusterAvailableResources; @@ -286,6 +287,11 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> { .askList(askList).resourceBlacklistRequest(blacklistRequest) .releaseList(releaseList).updateRequests(updateList).build(); + if (this.newTrackingUrl != null) { + allocateRequest.setTrackingUrl(this.newTrackingUrl); + this.appTrackingUrl = this.newTrackingUrl; + this.newTrackingUrl = null; + } // clear blacklistAdditions and blacklistRemovals before // unsynchronized part blacklistAdditions.clear(); @@ -930,6 +936,11 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> { } } + @Override + public synchronized void updateTrackingUrl(String trackingUrl) { + this.newTrackingUrl = trackingUrl; + } + private void updateAMRMToken(Token token) throws IOException { org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 9f5bad4..ff8b27c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -20,10 +20,12 @@ package org.apache.hadoop.yarn.client.api.impl; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; @@ -87,6 +89,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.mockito.ArgumentCaptor; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.mortbay.log.Log; @@ -2053,4 +2056,78 @@ public class TestAMRMClient { } return result; } + + @Test(timeout = 60000) + public void testNoUpdateTrackingUrl() { + try { + AMRMClientImpl<ContainerRequest> amClient = null; + amClient = new AMRMClientImpl<>(); + amClient.init(conf); + amClient.start(); + amClient.registerApplicationMaster("Host", 10000, ""); + + assertEquals("", amClient.appTrackingUrl); + + ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class); + AllocateResponse mockResponse = mock(AllocateResponse.class); + when(mockRM.allocate(any(AllocateRequest.class))) + .thenReturn(mockResponse); + ApplicationMasterProtocol realRM = amClient.rmClient; + amClient.rmClient = mockRM; + // Do allocate without updated tracking url + amClient.allocate(0.1f); + ArgumentCaptor<AllocateRequest> argument = + ArgumentCaptor.forClass(AllocateRequest.class); + verify(mockRM).allocate(argument.capture()); + assertNull(argument.getValue().getTrackingUrl()); + + amClient.rmClient = realRM; + amClient + .unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, + null); + } catch (IOException | YarnException e) { + throw new AssertionError( + "testNoUpdateTrackingUrl unexpectedly threw exception: " + e); + } + } + + @Test(timeout = 60000) + public void testUpdateTrackingUrl() { + try { + AMRMClientImpl<ContainerRequest> amClient = null; + amClient = new AMRMClientImpl<>(); + amClient.init(conf); + amClient.start(); + amClient.registerApplicationMaster("Host", 10000, ""); + + String trackingUrl = "hadoop.apache.org"; + assertEquals("", amClient.appTrackingUrl); + + ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class); + AllocateResponse mockResponse = mock(AllocateResponse.class); + when(mockRM.allocate(any(AllocateRequest.class))) + .thenReturn(mockResponse); + ApplicationMasterProtocol realRM = amClient.rmClient; + amClient.rmClient = mockRM; + // Do allocate with updated tracking url + amClient.updateTrackingUrl(trackingUrl); + assertEquals(trackingUrl, amClient.newTrackingUrl); + assertEquals("", amClient.appTrackingUrl); + amClient.allocate(0.1f); + assertNull(amClient.newTrackingUrl); + assertEquals(trackingUrl, amClient.appTrackingUrl); + ArgumentCaptor<AllocateRequest> argument + = ArgumentCaptor.forClass(AllocateRequest.class); + verify(mockRM).allocate(argument.capture()); + assertEquals(trackingUrl, argument.getValue().getTrackingUrl()); + + amClient.rmClient = realRM; + amClient + .unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, + null); + } catch (IOException | YarnException e) { + throw new AssertionError( + "testUpdateTrackingUrl unexpectedly threw exception: " + e); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java index f441d69..467ac1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java @@ -57,6 +57,7 @@ public class AllocateRequestPBImpl extends AllocateRequest { private List<ContainerId> release = null; private List<UpdateContainerRequest> updateRequests = null; private ResourceBlacklistRequest blacklistRequest = null; + private String trackingUrl = null; // This is deprecated, leave it here only to make unit test not break @Deprecated @@ -114,6 +115,9 @@ public class AllocateRequestPBImpl extends AllocateRequest { if (this.deprecatedIncreaseReqs != null) { addIncreaseRequestsToProto(); } + if (this.trackingUrl != null) { + builder.setTrackingUrl(this.trackingUrl); + } } private void mergeLocalToProto() { @@ -384,7 +388,28 @@ public class AllocateRequestPBImpl extends AllocateRequest { this.release.add(convertFromProtoFormat(c)); } } - + + @Override + public String getTrackingUrl() { + AllocateRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.trackingUrl != null) { + return this.trackingUrl; + } + if (p.hasTrackingUrl()) { + this.trackingUrl = p.getTrackingUrl(); + } + return this.trackingUrl; + } + + @Override + public void setTrackingUrl(String trackingUrl) { + maybeInitBuilder(); + if (trackingUrl == null) { + builder.clearTrackingUrl(); + } + this.trackingUrl = trackingUrl; + } + private void addReleasesToProto() { maybeInitBuilder(); builder.clearRelease(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index d5444b4..eea42e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -356,7 +356,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { // Send the status update to the appAttempt. getRmContext().getDispatcher().getEventHandler().handle( new RMAppAttemptStatusupdateEvent(appAttemptId, request - .getProgress())); + .getProgress(), request.getTrackingUrl())); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 07f949d..0b56f09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1815,6 +1815,28 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { // Update progress appAttempt.progress = statusUpdateEvent.getProgress(); + // Update tracking url if changed and save it to state store + String newTrackingUrl = statusUpdateEvent.getTrackingUrl(); + if (newTrackingUrl != null && + !newTrackingUrl.equals(appAttempt.originalTrackingUrl)) { + appAttempt.originalTrackingUrl = newTrackingUrl; + AggregateAppResourceUsage resUsage = + appAttempt.attemptMetrics.getAggregateAppResourceUsage(); + ApplicationAttemptStateData attemptState = ApplicationAttemptStateData + .newInstance(appAttempt.applicationAttemptId, + appAttempt.getMasterContainer(), + appAttempt.rmContext.getStateStore() + .getCredentialsFromAppAttempt(appAttempt), + appAttempt.startTime, appAttempt.recoveredFinalState, + newTrackingUrl, appAttempt.getDiagnostics(), null, + ContainerExitStatus.INVALID, appAttempt.getFinishTime(), + resUsage.getMemorySeconds(), resUsage.getVcoreSeconds(), + appAttempt.attemptMetrics.getPreemptedMemory(), + appAttempt.attemptMetrics.getPreemptedVcore()); + appAttempt.rmContext.getStateStore() + .updateApplicationAttemptState(attemptState); + } + // Ping to AMLivelinessMonitor appAttempt.rmContext.getAMLivelinessMonitor().receivedPing( statusUpdateEvent.getApplicationAttemptId()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStatusupdateEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStatusupdateEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStatusupdateEvent.java index b1b63b1..1b7442d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStatusupdateEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStatusupdateEvent.java @@ -25,15 +25,26 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE public class RMAppAttemptStatusupdateEvent extends RMAppAttemptEvent { private final float progress; + private final String trackingUrl; public RMAppAttemptStatusupdateEvent(ApplicationAttemptId appAttemptId, float progress) { + this(appAttemptId, progress, null); + } + + public RMAppAttemptStatusupdateEvent(ApplicationAttemptId appAttemptId, + float progress, String trackingUrl) { super(appAttemptId, RMAppAttemptEventType.STATUS_UPDATE); this.progress = progress; + this.trackingUrl = trackingUrl; } public float getProgress() { return this.progress; } + public String getTrackingUrl() { + return this.trackingUrl; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java index e684f3c..65e517d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java @@ -672,4 +672,38 @@ public class TestApplicationMasterService { Assert.fail("Cannot find RMContainer"); } } + + @Test(timeout = 300000) + public void testUpdateTrackingUrl() throws Exception { + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm = new MockRM(conf); + rm.start(); + + // Register node1 + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); + + RMApp app1 = rm.submitApp(2048); + + nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); + am1.registerAppAttempt(); + Assert.assertEquals("N/A", rm.getRMContext().getRMApps().get( + app1.getApplicationId()).getOriginalTrackingUrl()); + + AllocateRequestPBImpl allocateRequest = new AllocateRequestPBImpl(); + String newTrackingUrl = "hadoop.apache.org"; + allocateRequest.setTrackingUrl(newTrackingUrl); + + am1.allocate(allocateRequest); + Assert.assertEquals(newTrackingUrl, rm.getRMContext().getRMApps().get( + app1.getApplicationId()).getOriginalTrackingUrl()); + + // Send it again + am1.allocate(allocateRequest); + Assert.assertEquals(newTrackingUrl, rm.getRMContext().getRMApps().get( + app1.getApplicationId()).getOriginalTrackingUrl()); + rm.stop(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index da8c83c..f8fc402 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -2698,6 +2698,51 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { rm2.stop(); } + @Test(timeout = 20000) + public void testRMRestartAfterUpdateTrackingUrl() throws Exception { + MockRM rm = new MockRM(conf); + rm.start(); + + MemoryRMStateStore memStore = (MemoryRMStateStore) rm.getRMStateStore(); + + // Register node1 + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024); + + RMApp app1 = rm.submitApp(2048); + + nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); + am1.registerAppAttempt(); + + AllocateRequestPBImpl allocateRequest = new AllocateRequestPBImpl(); + String newTrackingUrl = "hadoop.apache.org"; + allocateRequest.setTrackingUrl(newTrackingUrl); + + am1.allocate(allocateRequest); + // Check in-memory and stored tracking url + Assert.assertEquals(newTrackingUrl, rm.getRMContext().getRMApps().get( + app1.getApplicationId()).getOriginalTrackingUrl()); + Assert.assertEquals(newTrackingUrl, rm.getRMContext().getRMApps().get( + app1.getApplicationId()).getCurrentAppAttempt() + .getOriginalTrackingUrl()); + Assert.assertEquals(newTrackingUrl, memStore.getState() + .getApplicationState().get(app1.getApplicationId()) + .getAttempt(attempt1.getAppAttemptId()).getFinalTrackingUrl()); + + // Start new RM, should recover updated tracking url + MockRM rm2 = new MockRM(conf, memStore); + rm2.start(); + Assert.assertEquals(newTrackingUrl, rm.getRMContext().getRMApps().get( + app1.getApplicationId()).getOriginalTrackingUrl()); + Assert.assertEquals(newTrackingUrl, rm.getRMContext().getRMApps().get( + app1.getApplicationId()).getCurrentAppAttempt() + .getOriginalTrackingUrl()); + + rm.stop(); + rm2.stop(); + } + private Credentials getCreds() throws IOException { Credentials ts = new Credentials(); DataOutputBuffer dob = new DataOutputBuffer(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
