Author: vinodkv
Date: Wed Mar 26 03:03:17 2014
New Revision: 1581662
URL: http://svn.apache.org/r1581662
Log:
YARN-1867. Fixed a bug in ResourceManager that was causing invalid ACL checks
in the web-services after fail-over. Contributed by Vinod Kumar Vavilapalli.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1581662&r1=1581661&r2=1581662&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed Mar 26 03:03:17 2014
@@ -577,6 +577,9 @@ Release 2.4.0 - UNRELEASED
YARN-1866. Fixed an issue with renewal of RM-delegation tokens on restart
or
fail-over. (Jian He via vinodkv)
+ YARN-1867. Fixed a bug in ResourceManager that was causing invalid ACL
checks
+ in the web-services after fail-over. (Vinod Kumar Vavilapalli)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1581662&r1=1581661&r2=1581662&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
Wed Mar 26 03:03:17 2014
@@ -81,6 +81,7 @@ public class RMContextImpl implements RM
private ApplicationMasterService applicationMasterService;
private RMApplicationHistoryWriter rmApplicationHistoryWriter;
private ConfigurationProvider configurationProvider;
+
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1581662&r1=1581661&r2=1581662&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Wed Mar 26 03:03:17 2014
@@ -141,19 +141,14 @@ public class ResourceManager extends Com
protected ResourceScheduler scheduler;
private ClientRMService clientRM;
protected ApplicationMasterService masterService;
- private ApplicationMasterLauncher applicationMasterLauncher;
- private ContainerAllocationExpirer containerAllocationExpirer;
protected NMLivelinessMonitor nmLivelinessMonitor;
protected NodesListManager nodesListManager;
- private EventHandler<SchedulerEvent> schedulerDispatcher;
protected RMAppManager rmAppManager;
protected ApplicationACLsManager applicationACLsManager;
protected QueueACLsManager queueACLsManager;
- private DelegationTokenRenewer delegationTokenRenewer;
private WebApp webApp;
private AppReportFetcher fetcher = null;
protected ResourceTrackerService resourceTracker;
- private boolean recoveryEnabled;
private String webAppAddress;
private ConfigurationProvider configurationProvider = null;
@@ -333,6 +328,14 @@ public class ResourceManager extends Com
*/
@Private
class RMActiveServices extends CompositeService {
+
+ private DelegationTokenRenewer delegationTokenRenewer;
+ private EventHandler<SchedulerEvent> schedulerDispatcher;
+ private ApplicationMasterLauncher applicationMasterLauncher;
+ private ContainerAllocationExpirer containerAllocationExpirer;
+
+ private boolean recoveryEnabled;
+
RMActiveServices() {
super("RMActiveServices");
}
@@ -1009,6 +1012,11 @@ public class ResourceManager extends Com
return this.queueACLsManager;
}
+ @Private
+ WebApp getWebapp() {
+ return this.webApp;
+ }
+
@Override
public void recover(RMState state) throws Exception {
// recover RMdelegationTokenSecretManager
@@ -1055,16 +1063,14 @@ public class ResourceManager extends Com
rmContext.setDispatcher(rmDispatcher);
}
-
/**
* Retrieve RM bind address from configuration
- *
+ *
* @param conf
* @return InetSocketAddress
*/
-public static InetSocketAddress getBindAddress(Configuration conf) {
+ public static InetSocketAddress getBindAddress(Configuration conf) {
return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
+ YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
}
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1581662&r1=1581661&r2=1581662&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
Wed Mar 26 03:03:17 2014
@@ -101,19 +101,12 @@ public class RMWebServices {
private final ResourceManager rm;
private static RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
- private final ApplicationACLsManager aclsManager;
- private final QueueACLsManager queueACLsManager;
private final Configuration conf;
private @Context HttpServletResponse response;
@Inject
- public RMWebServices(final ResourceManager rm,
- final ApplicationACLsManager aclsManager,
- final QueueACLsManager queueACLsManager,
- Configuration conf) {
+ public RMWebServices(final ResourceManager rm, Configuration conf) {
this.rm = rm;
- this.aclsManager = aclsManager;
- this.queueACLsManager = queueACLsManager;
this.conf = conf;
}
@@ -125,10 +118,11 @@ public class RMWebServices {
callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
}
if (callerUGI != null
- && !(this.aclsManager.checkAccess(callerUGI,
- ApplicationAccessType.VIEW_APP, app.getUser(),
- app.getApplicationId()) || this.queueACLsManager.checkAccess(
- callerUGI, QueueACL.ADMINISTER_QUEUE, app.getQueue()))) {
+ && !(this.rm.getApplicationACLsManager().checkAccess(callerUGI,
+ ApplicationAccessType.VIEW_APP, app.getUser(),
+ app.getApplicationId()) ||
+ this.rm.getQueueACLsManager().checkAccess(callerUGI,
+ QueueACL.ADMINISTER_QUEUE, app.getQueue()))) {
return false;
}
return true;
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1581662&r1=1581661&r2=1581662&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
Wed Mar 26 03:03:17 2014
@@ -80,6 +80,8 @@ import org.apache.log4j.Logger;
@SuppressWarnings("unchecked")
public class MockRM extends ResourceManager {
+ static final String ENABLE_WEBAPP = "mockrm.webapp.enabled";
+
public MockRM() {
this(new YarnConfiguration());
}
@@ -494,7 +496,12 @@ public class MockRM extends ResourceMana
@Override
protected void startWepApp() {
- // override to disable webapp
+ if (getConfig().getBoolean(ENABLE_WEBAPP, false)) {
+ super.startWepApp();
+ return;
+ }
+
+ // Disable webapp
}
public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM
nm,
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1581662&r1=1581661&r2=1581662&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
Wed Mar 26 03:03:17 2014
@@ -18,6 +18,18 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import javax.ws.rs.core.MediaType;
+
+import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -25,10 +37,11 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HealthCheckFailedException;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -36,17 +49,15 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
-
-import org.junit.Assert;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
public class TestRMHA {
private Log LOG = LogFactory.getLog(TestRMHA.class);
@@ -77,6 +88,10 @@ public class TestRMHA {
configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
}
+
+ // Enable webapp to test web-services also
+ configuration.setBoolean(MockRM.ENABLE_WEBAPP, true);
+ configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
}
private void checkMonitorHealth() throws IOException {
@@ -97,7 +112,7 @@ public class TestRMHA {
rm.adminService.getServiceStatus().isReadyToBecomeActive());
}
- private void checkActiveRMFunctionality() throws IOException {
+ private void checkActiveRMFunctionality() throws Exception {
assertEquals(STATE_ERR, HAServiceState.ACTIVE,
rm.adminService.getServiceStatus().getState());
assertTrue("Active RM services aren't started",
@@ -115,6 +130,33 @@ public class TestRMHA {
fail("Unable to perform Active RM functions");
LOG.error("ActiveRM check failed", e);
}
+
+ checkActiveRMWebServices();
+ }
+
+ // Do some sanity testing of the web-services after fail-over.
+ private void checkActiveRMWebServices() throws JSONException {
+
+ // Validate web-service
+ Client webServiceClient = Client.create(new DefaultClientConfig());
+ InetSocketAddress rmWebappAddr =
+ NetUtils.getConnectAddress(rm.getWebapp().getListenerAddress());
+ String webappURL =
+ "http://"; + rmWebappAddr.getHostName() + ":" + rmWebappAddr.getPort();
+ WebResource webResource = webServiceClient.resource(webappURL);
+ String path = app.getApplicationId().toString();
+
+ ClientResponse response =
+ webResource.path("ws").path("v1").path("cluster").path("apps")
+ .path(path).accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject appJson = json.getJSONObject("app");
+ assertEquals("ACCEPTED", appJson.getString("state"));
+ // Other stuff is verified in the regular web-services related tests
}
/**
@@ -129,9 +171,10 @@ public class TestRMHA {
* become Active
*/
@Test (timeout = 30000)
- public void testStartAndTransitions() throws Exception {
+ public void testFailoverAndTransitions() throws Exception {
configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
Configuration conf = new YarnConfiguration(configuration);
+
rm = new MockRM(conf);
rm.init(conf);
StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
@@ -191,7 +234,7 @@ public class TestRMHA {
}
@Test
- public void testTransitionsWhenAutomaticFailoverEnabled() throws IOException
{
+ public void testTransitionsWhenAutomaticFailoverEnabled() throws Exception {
final String ERR_UNFORCED_REQUEST = "User request succeeded even when " +
"automatic failover is enabled";
