Author: vinodkv
Date: Thu Sep 26 04:18:28 2013
New Revision: 1526372
URL: http://svn.apache.org/r1526372
Log:
YARN-1157. Fixed ResourceManager UI to behave correctly when apps like
distributed-shell do not set tracking urls. Contributed by Xuan Gong.
svn merge --ignore-ancestry -c 1526371 ../../trunk/
Removed:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1526372&r1=1526371&r2=1526372&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu Sep 26
04:18:28 2013
@@ -68,6 +68,9 @@ Release 2.1.2 - UNRELEASED
YARN-49. Improve distributed shell application to work on a secure cluster.
(Vinod Kumar Vavilapalli via hitesh)
+ YARN-1157. Fixed ResourceManager UI to behave correctly when apps like
+ distributed-shell do not set tracking urls. (Xuan Gong via vinodkv)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java?rev=1526372&r1=1526371&r2=1526372&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
Thu Sep 26 04:18:28 2013
@@ -100,11 +100,22 @@ public abstract class FinishApplicationM
public abstract String getTrackingUrl();
/**
- * Set the <em>tracking URL</em>for the <code>ApplicationMaster</code>
- * This url if contains scheme then that will be used by resource manager
- * web application proxy otherwise it will default to http.
- * @param url <em>tracking URL</em>for the
- * <code>ApplicationMaster</code>
+ * Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
+ * This is the web-URL to which ResourceManager or web-application proxy will
+ * redirect client/users once the application is finished and the
+ * <code>ApplicationMaster</code> is gone.
+ * <p>
+ * If the passed url has a scheme then that will be used by the
+ * ResourceManager and web-application proxy, otherwise the scheme will
+ * default to http.
+ * </p>
+ * <p>
+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an
url
+ * isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ * <p>
+ *
+ * @param url
+ * <em>tracking URL</em>for the <code>ApplicationMaster</code>
*/
@Public
@Stable
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java?rev=1526372&r1=1526371&r2=1526372&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
Thu Sep 26 04:18:28 2013
@@ -112,11 +112,22 @@ public abstract class RegisterApplicatio
public abstract String getTrackingUrl();
/**
- * Set the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
- * This url if contains scheme then that will be used by resource manager
- * web application proxy otherwise it will default to http.
- * @param trackingUrl <em>tracking URL</em> for the
- * <code>ApplicationMaster</code>
+ * Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
+ * it is running. This is the web-URL to which ResourceManager or
+ * web-application proxy will redirect client/users while the application and
+ * the <code>ApplicationMaster</code> are still running.
+ * <p>
+ * If the passed url has a scheme then that will be used by the
+ * ResourceManager and web-application proxy, otherwise the scheme will
+ * default to http.
+ * </p>
+ * <p>
+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an
url
+ * isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ * <p>
+ *
+ * @param trackingUrl
+ * <em>tracking URL</em>for the <code>ApplicationMaster</code>
*/
@Public
@Stable
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1526372&r1=1526371&r2=1526372&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
Thu Sep 26 04:18:28 2013
@@ -994,7 +994,7 @@ public class RMAppAttemptImpl implements
}
}
- static final class AMRegisteredTransition extends BaseTransition {
+ private static final class AMRegisteredTransition extends BaseTransition {
@Override
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
@@ -1003,7 +1003,8 @@ public class RMAppAttemptImpl implements
= (RMAppAttemptRegistrationEvent) event;
appAttempt.host = registrationEvent.getHost();
appAttempt.rpcPort = registrationEvent.getRpcport();
- appAttempt.origTrackingUrl = registrationEvent.getTrackingurl();
+ appAttempt.origTrackingUrl =
+ sanitizeTrackingUrl(registrationEvent.getTrackingurl());
appAttempt.proxiedTrackingUrl =
appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
@@ -1138,7 +1139,8 @@ public class RMAppAttemptImpl implements
RMAppAttemptUnregistrationEvent unregisterEvent
= (RMAppAttemptUnregistrationEvent) event;
appAttempt.diagnostics.append(unregisterEvent.getDiagnostics());
- appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl();
+ appAttempt.origTrackingUrl =
+ sanitizeTrackingUrl(unregisterEvent.getTrackingUrl());
appAttempt.proxiedTrackingUrl =
appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus();
@@ -1292,4 +1294,8 @@ public class RMAppAttemptImpl implements
appAttempt.rmContext.getAMRMTokenSecretManager()
.applicationMasterFinished(appAttempt.getAppAttemptId());
}
+
+ private static String sanitizeTrackingUrl(String url) {
+ return (url == null || url.trim().isEmpty()) ? "N/A" : url;
+ }
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1526372&r1=1526371&r2=1526372&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Thu Sep 26 04:18:28 2013
@@ -30,14 +30,18 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -85,8 +89,10 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import
org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -261,8 +267,22 @@ public class TestRMAppAttemptTransitions
private String getProxyUrl(RMAppAttempt appAttempt) {
- return pjoin(RM_WEBAPP_ADDR, "proxy",
- appAttempt.getAppAttemptId().getApplicationId(), "");
+ String url = null;
+ try {
+ URI trackingUri =
+ StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null :
+ ProxyUriUtils
+ .getUriFromAMUrl(appAttempt.getOriginalTrackingUrl());
+ String proxy = WebAppUtils.getProxyHostAndPort(conf);
+ URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
+ URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
+ appAttempt.getAppAttemptId().getApplicationId());
+ url = result.toASCIIString().substring(
+ HttpConfig.getSchemePrefix().length());
+ } catch (URISyntaxException ex) {
+ Assert.fail();
+ }
+ return url;
}
/**
@@ -448,9 +468,9 @@ public class TestRMAppAttemptTransitions
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(host, applicationAttempt.getHost());
assertEquals(rpcPort, applicationAttempt.getRpcPort());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
if (unmanagedAM) {
- assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
} else {
assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
@@ -468,7 +488,7 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.FINISHING,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
assertEquals(container, applicationAttempt.getMasterContainer());
@@ -487,9 +507,9 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.FINISHED,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
if (unmanagedAM) {
- assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
} else {
assertEquals(getProxyUrl(applicationAttempt),
@@ -603,9 +623,7 @@ public class TestRMAppAttemptTransitions
trackingUrl, diagnostics);
}
-
- @Test
- public void testUnmanagedAMSuccess() {
+ private void testUnmanagedAMSuccess(String url) {
unmanagedAM = true;
when(submissionContext.getUnmanagedAM()).thenReturn(true);
// submit AM and check it goes to LAUNCHED state
@@ -615,7 +633,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId());
// launch AM
- runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true);
+ runApplicationAttempt(null, "host", 8042, url, true);
// complete a container
applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent(
@@ -623,13 +641,12 @@ public class TestRMAppAttemptTransitions
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class)));
// complete AM
- String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
- applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
+ applicationAttempt.getAppAttemptId(), url, finalStatus,
diagnostics));
- testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1,
+ testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
true);
}
@@ -824,12 +841,42 @@ public class TestRMAppAttemptTransitions
"Killed by user");
}
+ @Test
+ public void testTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess("oldTrackingUrl");
+ }
@Test
- public void testNoTrackingUrl() {
+ public void testEmptyTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess("");
+ }
+
+ @Test
+ public void testNullTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess(null);
+ }
+
+ @Test
+ public void testManagedAMWithTrackingUrl() {
+ testTrackingUrlManagedAM("theTrackingUrl");
+ }
+
+ @Test
+ public void testManagedAMWithEmptyTrackingUrl() {
+ testTrackingUrlManagedAM("");
+ }
+
+ @Test
+ public void testManagedAMWithNullTrackingUrl() {
+ testTrackingUrlManagedAM(null);
+ }
+
+ private void testTrackingUrlManagedAM(String url) {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "", false);
+ runApplicationAttempt(amContainer, "host", 8042, url, false);
+ unregisterApplicationAttempt(amContainer,
+ FinalApplicationStatus.SUCCEEDED, url, "Successful");
}
@Test
@@ -927,4 +974,12 @@ public class TestRMAppAttemptTransitions
}
}
}
+
+ private void verifyUrl(String url1, String url2) {
+ if (url1 == null || url1.trim().isEmpty()) {
+ assertEquals("N/A", url2);
+ } else {
+ assertEquals(url1, url2);
+ }
+ }
}
