Repository: hadoop Updated Branches: refs/heads/trunk 572cdb546 -> 72d22b753
YARN-7207. Cache the RM proxy server address. (Yufei Gu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72d22b75 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72d22b75 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72d22b75 Branch: refs/heads/trunk Commit: 72d22b753abde4d07a727479d3f3d5d84d5dd6b2 Parents: 572cdb5 Author: Yufei Gu <yu...@apache.org> Authored: Fri Oct 6 09:45:10 2017 -0700 Committer: Yufei Gu <yu...@apache.org> Committed: Fri Oct 6 09:49:11 2017 -0700 ---------------------------------------------------------------------- .../yarn/server/resourcemanager/RMContext.java | 2 ++ .../server/resourcemanager/RMContextImpl.java | 37 ++++++++++++++++++-- .../server/resourcemanager/ResourceManager.java | 2 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 20 +---------- .../rmapp/attempt/RMAppAttemptImpl.java | 23 ++---------- .../attempt/TestRMAppAttemptTransitions.java | 20 +++-------- 6 files changed, 45 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 6df3135..ec94030 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -164,4 +164,6 @@ public interface RMContext extends ApplicationMasterServiceContext { ResourceProfilesManager getResourceProfilesManager(); void setResourceProfilesManager(ResourceProfilesManager mgr); + + String getAppProxyUrl(Configuration conf, ApplicationId applicationId); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 921e5d5..80a9109 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -18,9 +18,13 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -53,9 +57,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManag import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; +import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.util.Clock; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; /** * RMContextImpl class holds two services context. @@ -71,6 +77,8 @@ import com.google.common.annotations.VisibleForTesting; */ public class RMContextImpl implements RMContext { + private static final Log LOG = LogFactory.getLog(RMContextImpl.class); + private static final String UNAVAILABLE = "N/A"; /** * RM service contexts which runs through out RM life span. These are created * once during start of RM. @@ -85,6 +93,8 @@ public class RMContextImpl implements RMContext { private ResourceProfilesManager resourceProfilesManager; + private String proxyHostAndPort = null; + /** * Default constructor. To be used in conjunction with setter methods for * individual fields. @@ -118,7 +128,7 @@ public class RMContextImpl implements RMContext { ConfigurationProvider provider = new LocalConfigurationProvider(); setConfigurationProvider(provider); } - + @VisibleForTesting // helper constructor for tests public RMContextImpl(Dispatcher rmDispatcher, @@ -527,7 +537,7 @@ public class RMContextImpl implements RMContext { public PlacementManager getQueuePlacementManager() { return this.activeServiceContext.getQueuePlacementManager(); } - + @Override public void setQueuePlacementManager(PlacementManager placementMgr) { this.activeServiceContext.setQueuePlacementManager(placementMgr); @@ -559,6 +569,29 @@ public class RMContextImpl implements RMContext { return this.resourceProfilesManager; } + String getProxyHostAndPort(Configuration conf) { + if (proxyHostAndPort == null) { + proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf); + } + return proxyHostAndPort; + } + + @Override + public String getAppProxyUrl(Configuration conf, ApplicationId applicationId) + { + try { + final String scheme = WebAppUtils.getHttpSchemePrefix(conf); + URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, + getProxyHostAndPort(conf)); + URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId); + return result.toASCIIString(); + } catch(URISyntaxException e) { + LOG.warn("Could not generate default proxy tracking URL for " + + applicationId); + return UNAVAILABLE; + } + } + @Override public void setResourceProfilesManager(ResourceProfilesManager mgr) { this.resourceProfilesManager = mgr; http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index e53a42c..fd95ee5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1077,7 +1077,7 @@ public class ResourceManager extends CompositeService implements Recoverable { .withCSRFProtection(YarnConfiguration.RM_CSRF_PREFIX) .withXFSProtection(YarnConfiguration.RM_XFS_PREFIX) .at(webAppAddress); - String proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf); + String proxyHostAndPort = rmContext.getProxyHostAndPort(conf); if(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf). equals(proxyHostAndPort)) { if (HAUtil.isHAEnabled(conf)) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 09381f9..cfb8a74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.net.InetAddress; -import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -103,7 +101,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSch import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.state.InvalidStateTransitionException; import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition; @@ -113,7 +110,6 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -707,7 +703,7 @@ public class RMAppImpl implements RMApp, Recoverable { float progress = 0.0f; org.apache.hadoop.yarn.api.records.Token amrmToken = null; if (allowAccess) { - trackingUrl = getDefaultProxyTrackingUrl(); + trackingUrl = rmContext.getAppProxyUrl(conf, applicationId); if (this.currentAttempt != null) { currentApplicationAttemptId = this.currentAttempt.getAppAttemptId(); trackingUrl = this.currentAttempt.getTrackingUrl(); @@ -797,20 +793,6 @@ public class RMAppImpl implements RMApp, Recoverable { } } - private String getDefaultProxyTrackingUrl() { - try { - final String scheme = WebAppUtils.getHttpSchemePrefix(conf); - String proxy = WebAppUtils.getProxyHostAndPort(conf); - URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy); - URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId); - return result.toASCIIString(); - } catch (URISyntaxException e) { - LOG.warn("Could not generate default proxy tracking URL for " - + applicationId); - return UNAVAILABLE; - } - } - @Override public long getFinishTime() { this.readLock.lock(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1b4d613..4997bc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; import static org.apache.hadoop.yarn.util.StringHelper.pjoin; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -521,7 +519,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); - this.proxiedTrackingUrl = generateProxyUriWithScheme(); + this.proxiedTrackingUrl = rmContext.getAppProxyUrl(conf, + appAttemptId.getApplicationId()); this.stateMachine = stateMachineFactory.make(this); this.attemptMetrics = @@ -654,24 +653,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - private String generateProxyUriWithScheme() { - this.readLock.lock(); - try { - final String scheme = WebAppUtils.getHttpSchemePrefix(conf); - String proxy = WebAppUtils.getProxyHostAndPort(conf); - URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy); - URI result = ProxyUriUtils.getProxyUri(null, proxyUri, - applicationAttemptId.getApplicationId()); - return result.toASCIIString(); - } catch (URISyntaxException e) { - LOG.warn("Could not proxify the uri for " - + applicationAttemptId.getApplicationId(), e); - return null; - } finally { - this.readLock.unlock(); - } - } - private void setTrackingUrlToRMAppPage(RMAppAttemptState stateToBeStored) { originalTrackingUrl = pjoin( WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index f6406ff..9dd5703 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -35,8 +35,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -113,7 +111,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManag import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.MasterKeyData; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; @@ -344,19 +341,10 @@ public class TestRMAppAttemptTransitions { ((AsyncDispatcher)this.spyRMContext.getDispatcher()).stop(); } - private String getProxyUrl(RMAppAttempt appAttempt) { - String url = null; - final String scheme = WebAppUtils.getHttpSchemePrefix(conf); - try { - String proxy = WebAppUtils.getProxyHostAndPort(conf); - URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy); - URI result = ProxyUriUtils.getProxyUri(null, proxyUri, appAttempt - .getAppAttemptId().getApplicationId()); - url = result.toASCIIString(); - } catch (URISyntaxException ex) { - Assert.fail(); - } + String url = rmContext.getAppProxyUrl(conf, + appAttempt.getAppAttemptId().getApplicationId()); + Assert.assertNotEquals("N/A", url); return url; } @@ -552,7 +540,7 @@ public class TestRMAppAttemptTransitions { if (unmanagedAM) { verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl()); } else { - assertEquals(getProxyUrl(applicationAttempt), + assertEquals(getProxyUrl(applicationAttempt), applicationAttempt.getTrackingUrl()); } // TODO - need to add more checks relevant to this state --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
