Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java Tue Aug 19 23:49:39 2014 @@ -72,6 +72,9 @@ public class MetricsOverviewTable extend th().$class("ui-state-default")._("Memory Used")._(). th().$class("ui-state-default")._("Memory Total")._(). th().$class("ui-state-default")._("Memory Reserved")._(). + th().$class("ui-state-default")._("VCores Used")._(). + th().$class("ui-state-default")._("VCores Total")._(). + th().$class("ui-state-default")._("VCores Reserved")._(). th().$class("ui-state-default")._("Active Nodes")._(). th().$class("ui-state-default")._("Decommissioned Nodes")._(). th().$class("ui-state-default")._("Lost Nodes")._(). @@ -94,6 +97,9 @@ public class MetricsOverviewTable extend td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)). + td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())). + td(String.valueOf(clusterMetrics.getTotalVirtualCores())). + td(String.valueOf(clusterMetrics.getReservedVirtualCores())). td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._(). td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._(). td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._(). @@ -120,6 +126,9 @@ public class MetricsOverviewTable extend th().$class("ui-state-default")._("Memory Used")._(). th().$class("ui-state-default")._("Memory Pending")._(). th().$class("ui-state-default")._("Memory Reserved")._(). + th().$class("ui-state-default")._("VCores Used")._(). + th().$class("ui-state-default")._("VCores Pending")._(). + th().$class("ui-state-default")._("VCores Reserved")._(). _(). _(). tbody().$class("ui-widget-content"). @@ -139,6 +148,9 @@ public class MetricsOverviewTable extend td(StringUtils.byteDesc(userMetrics.getAllocatedMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(userMetrics.getPendingMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)). + td(String.valueOf(userMetrics.getAllocatedVirtualCores())). + td(String.valueOf(userMetrics.getPendingVirtualCores())). + td(String.valueOf(userMetrics.getReservedVirtualCores())). _(). _()._();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Tue Aug 19 23:49:39 2014 @@ -75,6 +75,8 @@ class NodesPage extends RmView { th(".containers", "Containers"). th(".mem", "Mem Used"). th(".mem", "Mem Avail"). + th(".vcores", "VCores Used"). + th(".vcores", "VCores Avail"). th(".nodeManagerVersion", "Version"). _()._(). tbody(); @@ -127,8 +129,10 @@ class NodesPage extends RmView { td(String.valueOf(info.getNumContainers())). td().br().$title(String.valueOf(usedMemory))._(). _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._(). - td().br().$title(String.valueOf(usedMemory))._(). + td().br().$title(String.valueOf(availableMemory))._(). _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._(). + td(String.valueOf(info.getUsedVirtualCores())). + td(String.valueOf(info.getAvailableVirtualCores())). td(ni.getNodeManagerVersion()). _(); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Tue Aug 19 23:49:39 2014 @@ -19,6 +19,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.AccessControlException; +import java.nio.ByteBuffer; +import java.security.Principal; +import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; @@ -31,31 +36,70 @@ import java.util.concurrent.ConcurrentMa import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -66,23 +110,29 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; -import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -104,6 +154,9 @@ public class RMWebServices { private final Configuration conf; private @Context HttpServletResponse response; + public final static String DELEGATION_TOKEN_HEADER = + "Hadoop-YARN-RM-Delegation-Token"; + @Inject public RMWebServices(final ResourceManager rm, Configuration conf) { this.rm = rm; @@ -112,11 +165,7 @@ public class RMWebServices { protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) { // Check for the authorization. - String remoteUser = hsr.getRemoteUser(); - UserGroupInformation callerUGI = null; - if (remoteUser != null) { - callerUGI = UserGroupInformation.createRemoteUser(remoteUser); - } + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); if (callerUGI != null && !(this.rm.getApplicationACLsManager().checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), @@ -584,4 +633,679 @@ public class RMWebServices { return appAttemptsInfo; } + + @GET + @Path("/apps/{appid}/state") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public AppState getAppState(@Context HttpServletRequest hsr, + @PathParam("appid") String appId) throws AuthorizationException { + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + String userName = ""; + if (callerUGI != null) { + userName = callerUGI.getUserName(); + } + RMApp app = null; + try { + app = getRMAppForAppId(appId); + } catch (NotFoundException e) { + RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST, + "UNKNOWN", "RMWebService", + "Trying to get state of an absent application " + appId); + throw e; + } + + AppState ret = new AppState(); + ret.setState(app.getState().toString()); + + return ret; + } + + // can't return POJO because we can't control the status code + // it's always set to 200 when we need to allow it to be set + // to 202 + + @PUT + @Path("/apps/{appid}/state") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response updateAppState(AppState targetState, + @Context HttpServletRequest hsr, @PathParam("appid") String appId) + throws AuthorizationException, YarnException, InterruptedException, + IOException { + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + String userName = callerUGI.getUserName(); + RMApp app = null; + try { + app = getRMAppForAppId(appId); + } catch (NotFoundException e) { + RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST, + "UNKNOWN", "RMWebService", "Trying to kill/move an absent application " + + appId); + throw e; + } + + if (!app.getState().toString().equals(targetState.getState())) { + // user is attempting to change state. right we only + // allow users to kill the app + + if (targetState.getState().equals(YarnApplicationState.KILLED.toString())) { + return killApp(app, callerUGI, hsr); + } + throw new BadRequestException("Only '" + + YarnApplicationState.KILLED.toString() + + "' is allowed as a target state."); + } + + AppState ret = new AppState(); + ret.setState(app.getState().toString()); + + return Response.status(Status.OK).entity(ret).build(); + } + + protected Response killApp(RMApp app, UserGroupInformation callerUGI, + HttpServletRequest hsr) throws IOException, InterruptedException { + + if (app == null) { + throw new IllegalArgumentException("app cannot be null"); + } + String userName = callerUGI.getUserName(); + final ApplicationId appid = app.getApplicationId(); + KillApplicationResponse resp = null; + try { + resp = + callerUGI + .doAs(new PrivilegedExceptionAction<KillApplicationResponse>() { + @Override + public KillApplicationResponse run() throws IOException, + YarnException { + KillApplicationRequest req = + KillApplicationRequest.newInstance(appid); + return rm.getClientRMService().forceKillApplication(req); + } + }); + } catch (UndeclaredThrowableException ue) { + // if the root cause is a permissions issue + // bubble that up to the user + if (ue.getCause() instanceof YarnException) { + YarnException ye = (YarnException) ue.getCause(); + if (ye.getCause() instanceof AccessControlException) { + String appId = app.getApplicationId().toString(); + String msg = + "Unauthorized attempt to kill appid " + appId + + " by remote user " + userName; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } else { + throw ue; + } + } else { + throw ue; + } + } + + AppState ret = new AppState(); + ret.setState(app.getState().toString()); + + if (resp.getIsKillCompleted()) { + RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST, + "RMWebService", app.getApplicationId()); + } else { + return Response.status(Status.ACCEPTED).entity(ret) + .header(HttpHeaders.LOCATION, hsr.getRequestURL()).build(); + } + return Response.status(Status.OK).entity(ret).build(); + } + + private RMApp getRMAppForAppId(String appId) { + + if (appId == null || appId.isEmpty()) { + throw new NotFoundException("appId, " + appId + ", is empty or null"); + } + ApplicationId id; + try { + id = ConverterUtils.toApplicationId(recordFactory, appId); + } catch (NumberFormatException e) { + throw new NotFoundException("appId is invalid"); + } + if (id == null) { + throw new NotFoundException("appId is invalid"); + } + RMApp app = rm.getRMContext().getRMApps().get(id); + if (app == null) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + return app; + } + + private UserGroupInformation getCallerUserGroupInformation( + HttpServletRequest hsr, boolean usePrincipal) { + + String remoteUser = hsr.getRemoteUser(); + if (usePrincipal) { + Principal princ = hsr.getUserPrincipal(); + remoteUser = princ == null ? null : princ.getName(); + } + + UserGroupInformation callerUGI = null; + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } + + return callerUGI; + } + + private boolean isStaticUser(UserGroupInformation callerUGI) { + String staticUser = + conf.get(CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER, + CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER); + return staticUser.equals(callerUGI.getUserName()); + } + + /** + * Generates a new ApplicationId which is then sent to the client + * + * @param hsr + * the servlet request + * @return Response containing the app id and the maximum resource + * capabilities + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/apps/new-application") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response createNewApplication(@Context HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + NewApplication appId = createNewApplication(); + return Response.status(Status.OK).entity(appId).build(); + + } + + // reuse the code in ClientRMService to create new app + // get the new app id and submit app + // set location header with new app location + /** + * Function to submit an app to the RM + * + * @param newApp + * structure containing information to construct the + * ApplicationSubmissionContext + * @param hsr + * the servlet request + * @return Response containing the status code + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/apps") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response submitApplication(ApplicationSubmissionContextInfo newApp, + @Context HttpServletRequest hsr) throws AuthorizationException, + IOException, InterruptedException { + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + ApplicationSubmissionContext appContext = + createAppSubmissionContext(newApp); + final SubmitApplicationRequest req = + SubmitApplicationRequest.newInstance(appContext); + + try { + callerUGI + .doAs(new PrivilegedExceptionAction<SubmitApplicationResponse>() { + @Override + public SubmitApplicationResponse run() throws IOException, + YarnException { + return rm.getClientRMService().submitApplication(req); + } + }); + } catch (UndeclaredThrowableException ue) { + if (ue.getCause() instanceof YarnException) { + throw new BadRequestException(ue.getCause().getMessage()); + } + LOG.info("Submit app request failed", ue); + throw ue; + } + + String url = hsr.getRequestURL() + "/" + newApp.getApplicationId(); + return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, url) + .build(); + } + + /** + * Function that actually creates the ApplicationId by calling the + * ClientRMService + * + * @return returns structure containing the app-id and maximum resource + * capabilities + */ + private NewApplication createNewApplication() { + GetNewApplicationRequest req = + recordFactory.newRecordInstance(GetNewApplicationRequest.class); + GetNewApplicationResponse resp; + try { + resp = rm.getClientRMService().getNewApplication(req); + } catch (YarnException e) { + String msg = "Unable to create new app from RM web service"; + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); + } + NewApplication appId = + new NewApplication(resp.getApplicationId().toString(), + new ResourceInfo(resp.getMaximumResourceCapability())); + return appId; + } + + /** + * Create the actual ApplicationSubmissionContext to be submitted to the RM + * from the information provided by the user. + * + * @param newApp + * the information provided by the user + * @return returns the constructed ApplicationSubmissionContext + * @throws IOException + */ + protected ApplicationSubmissionContext createAppSubmissionContext( + ApplicationSubmissionContextInfo newApp) throws IOException { + + // create local resources and app submission context + + ApplicationId appid; + String error = + "Could not parse application id " + newApp.getApplicationId(); + try { + appid = + ConverterUtils.toApplicationId(recordFactory, + newApp.getApplicationId()); + } catch (Exception e) { + throw new BadRequestException(error); + } + ApplicationSubmissionContext appContext = + ApplicationSubmissionContext.newInstance(appid, + newApp.getApplicationName(), newApp.getQueue(), + Priority.newInstance(newApp.getPriority()), + createContainerLaunchContext(newApp), newApp.getUnmanagedAM(), + newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), + createAppSubmissionContextResource(newApp), + newApp.getApplicationType(), + newApp.getKeepContainersAcrossApplicationAttempts()); + appContext.setApplicationTags(newApp.getApplicationTags()); + + return appContext; + } + + protected Resource createAppSubmissionContextResource( + ApplicationSubmissionContextInfo newApp) throws BadRequestException { + if (newApp.getResource().getvCores() > rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) { + String msg = "Requested more cores than configured max"; + throw new BadRequestException(msg); + } + if (newApp.getResource().getMemory() > rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) { + String msg = "Requested more memory than configured max"; + throw new BadRequestException(msg); + } + Resource r = + Resource.newInstance(newApp.getResource().getMemory(), newApp + .getResource().getvCores()); + return r; + } + + /** + * Create the ContainerLaunchContext required for the + * ApplicationSubmissionContext. This function takes the user information and + * generates the ByteBuffer structures required by the ContainerLaunchContext + * + * @param newApp + * the information provided by the user + * @return created context + * @throws BadRequestException + * @throws IOException + */ + protected ContainerLaunchContext createContainerLaunchContext( + ApplicationSubmissionContextInfo newApp) throws BadRequestException, + IOException { + + // create container launch context + + HashMap<String, ByteBuffer> hmap = new HashMap<String, ByteBuffer>(); + for (Map.Entry<String, String> entry : newApp + .getContainerLaunchContextInfo().getAuxillaryServiceData().entrySet()) { + if (entry.getValue().isEmpty() == false) { + Base64 decoder = new Base64(0, null, true); + byte[] data = decoder.decode(entry.getValue()); + hmap.put(entry.getKey(), ByteBuffer.wrap(data)); + } + } + + HashMap<String, LocalResource> hlr = new HashMap<String, LocalResource>(); + for (Map.Entry<String, LocalResourceInfo> entry : newApp + .getContainerLaunchContextInfo().getResources().entrySet()) { + LocalResourceInfo l = entry.getValue(); + LocalResource lr = + LocalResource.newInstance( + ConverterUtils.getYarnUrlFromURI(l.getUrl()), l.getType(), + l.getVisibility(), l.getSize(), l.getTimestamp()); + hlr.put(entry.getKey(), lr); + } + + DataOutputBuffer out = new DataOutputBuffer(); + Credentials cs = + createCredentials(newApp.getContainerLaunchContextInfo() + .getCredentials()); + cs.writeTokenStorageToStream(out); + ByteBuffer tokens = ByteBuffer.wrap(out.getData()); + + ContainerLaunchContext ctx = + ContainerLaunchContext.newInstance(hlr, newApp + .getContainerLaunchContextInfo().getEnvironment(), newApp + .getContainerLaunchContextInfo().getCommands(), hmap, tokens, newApp + .getContainerLaunchContextInfo().getAcls()); + + return ctx; + } + + /** + * Generate a Credentials object from the information in the CredentialsInfo + * object. + * + * @param credentials + * the CredentialsInfo provided by the user. + * @return + */ + private Credentials createCredentials(CredentialsInfo credentials) { + Credentials ret = new Credentials(); + try { + for (Map.Entry<String, String> entry : credentials.getTokens().entrySet()) { + Text alias = new Text(entry.getKey()); + Token<TokenIdentifier> token = new Token<TokenIdentifier>(); + token.decodeFromUrlString(entry.getValue()); + ret.addToken(alias, token); + } + for (Map.Entry<String, String> entry : credentials.getTokens().entrySet()) { + Text alias = new Text(entry.getKey()); + Base64 decoder = new Base64(0, null, true); + byte[] secret = decoder.decode(entry.getValue()); + ret.addSecretKey(alias, secret); + } + } catch (IOException ie) { + throw new BadRequestException( + "Could not parse credentials data; exception message = " + + ie.getMessage()); + } + return ret; + } + + private UserGroupInformation createKerberosUserGroupInformation( + HttpServletRequest hsr) throws AuthorizationException, YarnException { + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + + String authType = hsr.getAuthType(); + if (!KerberosAuthenticationHandler.TYPE.equals(authType)) { + String msg = + "Delegation token operations can only be carried out on a " + + "Kerberos authenticated channel"; + throw new YarnException(msg); + } + + callerUGI.setAuthenticationMethod(AuthenticationMethod.KERBEROS); + return callerUGI; + } + + @POST + @Path("/delegation-token") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response postDelegationToken(DelegationToken tokenData, + @Context HttpServletRequest hsr) throws AuthorizationException, + IOException, InterruptedException, Exception { + + init(); + UserGroupInformation callerUGI; + try { + callerUGI = createKerberosUserGroupInformation(hsr); + } catch (YarnException ye) { + return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build(); + } + return createDelegationToken(tokenData, hsr, callerUGI); + } + + @POST + @Path("/delegation-token/expiration") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response + postDelegationTokenExpiration(@Context HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException, + Exception { + + init(); + UserGroupInformation callerUGI; + try { + callerUGI = createKerberosUserGroupInformation(hsr); + } catch (YarnException ye) { + return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build(); + } + + DelegationToken requestToken = new DelegationToken(); + requestToken.setToken(extractToken(hsr).encodeToUrlString()); + return renewDelegationToken(requestToken, hsr, callerUGI); + } + + private Response createDelegationToken(DelegationToken tokenData, + HttpServletRequest hsr, UserGroupInformation callerUGI) + throws AuthorizationException, IOException, InterruptedException, + Exception { + + final String renewer = tokenData.getRenewer(); + GetDelegationTokenResponse resp; + try { + resp = + callerUGI + .doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() { + @Override + public GetDelegationTokenResponse run() throws IOException, + YarnException { + GetDelegationTokenRequest createReq = + GetDelegationTokenRequest.newInstance(renewer); + return rm.getClientRMService().getDelegationToken(createReq); + } + }); + } catch (Exception e) { + LOG.info("Create delegation token request failed", e); + throw e; + } + + Token<RMDelegationTokenIdentifier> tk = + new Token<RMDelegationTokenIdentifier>(resp.getRMDelegationToken() + .getIdentifier().array(), resp.getRMDelegationToken().getPassword() + .array(), new Text(resp.getRMDelegationToken().getKind()), new Text( + resp.getRMDelegationToken().getService())); + RMDelegationTokenIdentifier identifier = tk.decodeIdentifier(); + long currentExpiration = + rm.getRMContext().getRMDelegationTokenSecretManager() + .getRenewDate(identifier); + DelegationToken respToken = + new DelegationToken(tk.encodeToUrlString(), renewer, identifier + .getOwner().toString(), tk.getKind().toString(), currentExpiration, + identifier.getMaxDate()); + return Response.status(Status.OK).entity(respToken).build(); + } + + private Response renewDelegationToken(DelegationToken tokenData, + HttpServletRequest hsr, UserGroupInformation callerUGI) + throws AuthorizationException, IOException, InterruptedException, + Exception { + + Token<RMDelegationTokenIdentifier> token = + extractToken(tokenData.getToken()); + + org.apache.hadoop.yarn.api.records.Token dToken = + BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind() + .toString(), token.getPassword(), token.getService().toString()); + final RenewDelegationTokenRequest req = + RenewDelegationTokenRequest.newInstance(dToken); + + RenewDelegationTokenResponse resp; + try { + resp = + callerUGI + .doAs(new PrivilegedExceptionAction<RenewDelegationTokenResponse>() { + @Override + public RenewDelegationTokenResponse run() throws IOException, + YarnException { + return rm.getClientRMService().renewDelegationToken(req); + } + }); + } catch (UndeclaredThrowableException ue) { + if (ue.getCause() instanceof YarnException) { + if (ue.getCause().getCause() instanceof InvalidToken) { + throw new BadRequestException(ue.getCause().getCause().getMessage()); + } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) { + return Response.status(Status.FORBIDDEN) + .entity(ue.getCause().getCause().getMessage()).build(); + } + LOG.info("Renew delegation token request failed", ue); + throw ue; + } + LOG.info("Renew delegation token request failed", ue); + throw ue; + } catch (Exception e) { + LOG.info("Renew delegation token request failed", e); + throw e; + } + long renewTime = resp.getNextExpirationTime(); + + DelegationToken respToken = new DelegationToken(); + respToken.setNextExpirationTime(renewTime); + return Response.status(Status.OK).entity(respToken).build(); + } + + // For cancelling tokens, the encoded token is passed as a header + // There are two reasons for this - + // 1. Passing a request body as part of a DELETE request is not + // allowed by Jetty + // 2. Passing the encoded token as part of the url is not ideal + // since urls tend to get logged and anyone with access to + // the logs can extract tokens which are meant to be secret + @DELETE + @Path("/delegation-token") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response cancelDelegationToken(@Context HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException, + Exception { + + init(); + UserGroupInformation callerUGI; + try { + callerUGI = createKerberosUserGroupInformation(hsr); + } catch (YarnException ye) { + return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build(); + } + + Token<RMDelegationTokenIdentifier> token = extractToken(hsr); + + org.apache.hadoop.yarn.api.records.Token dToken = + BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind() + .toString(), token.getPassword(), token.getService().toString()); + final CancelDelegationTokenRequest req = + CancelDelegationTokenRequest.newInstance(dToken); + + try { + callerUGI + .doAs(new PrivilegedExceptionAction<CancelDelegationTokenResponse>() { + @Override + public CancelDelegationTokenResponse run() throws IOException, + YarnException { + return rm.getClientRMService().cancelDelegationToken(req); + } + }); + } catch (UndeclaredThrowableException ue) { + if (ue.getCause() instanceof YarnException) { + if (ue.getCause().getCause() instanceof InvalidToken) { + throw new BadRequestException(ue.getCause().getCause().getMessage()); + } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) { + return Response.status(Status.FORBIDDEN) + .entity(ue.getCause().getCause().getMessage()).build(); + } + LOG.info("Renew delegation token request failed", ue); + throw ue; + } + LOG.info("Renew delegation token request failed", ue); + throw ue; + } catch (Exception e) { + LOG.info("Renew delegation token request failed", e); + throw e; + } + + return Response.status(Status.OK).build(); + } + + private Token<RMDelegationTokenIdentifier> extractToken( + HttpServletRequest request) { + String encodedToken = request.getHeader(DELEGATION_TOKEN_HEADER); + if (encodedToken == null) { + String msg = + "Header '" + DELEGATION_TOKEN_HEADER + + "' containing encoded token not found"; + throw new BadRequestException(msg); + } + return extractToken(encodedToken); + } + + private Token<RMDelegationTokenIdentifier> extractToken(String encodedToken) { + Token<RMDelegationTokenIdentifier> token = + new Token<RMDelegationTokenIdentifier>(); + try { + token.decodeFromUrlString(encodedToken); + } catch (Exception ie) { + String msg = "Could not decode encoded token"; + throw new BadRequestException(msg); + } + return token; + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java Tue Aug 19 23:49:39 2014 @@ -77,7 +77,7 @@ public class RmView extends TwoColumnLay StringBuilder sb = new StringBuilder(); return sb .append("[\n") - .append("{'sType':'numeric', 'aTargets': [0]") + .append("{'sType':'string', 'aTargets': [0]") .append(", 'mRender': parseHadoopID }") .append("\n, {'sType':'numeric', 'aTargets': [5, 6]") Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java Tue Aug 19 23:49:39 2014 @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; @@ -78,6 +79,12 @@ public class AppInfo { protected int allocatedMB; protected int allocatedVCores; protected int runningContainers; + + // preemption info fields + protected int preemptedResourceMB; + protected int preemptedResourceVCores; + protected int numNonAMContainerPreempted; + protected int numAMContainerPreempted; public AppInfo() { } // JAXB needs this @@ -147,6 +154,17 @@ public class AppInfo { } } } + + // copy preemption info fields + RMAppMetrics appMetrics = app.getRMAppMetrics(); + numAMContainerPreempted = + appMetrics.getNumAMContainersPreempted(); + preemptedResourceMB = + appMetrics.getResourcePreempted().getMemory(); + numNonAMContainerPreempted = + appMetrics.getNumNonAMContainersPreempted(); + preemptedResourceVCores = + appMetrics.getResourcePreempted().getVirtualCores(); } } @@ -254,4 +272,19 @@ public class AppInfo { return this.allocatedVCores; } + public int getPreemptedMB() { + return preemptedResourceMB; + } + + public int getPreemptedVCores() { + return preemptedResourceVCores; + } + + public int getNumNonAMContainersPreempted() { + return numNonAMContainerPreempted; + } + + public int getNumAMContainersPreempted() { + return numAMContainerPreempted; + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java Tue Aug 19 23:49:39 2014 @@ -37,16 +37,21 @@ public class ClusterMetricsInfo { protected int appsRunning; protected int appsFailed; protected int appsKilled; - + protected long reservedMB; protected long availableMB; protected long allocatedMB; - + + protected long reservedVirtualCores; + protected long availableVirtualCores; + protected long allocatedVirtualCores; + protected int containersAllocated; protected int containersReserved; protected int containersPending; - + protected long totalMB; + protected long totalVirtualCores; protected int totalNodes; protected int lostNodes; protected int unhealthyNodes; @@ -68,16 +73,21 @@ public class ClusterMetricsInfo { this.appsRunning = metrics.getAppsRunning(); this.appsFailed = metrics.getAppsFailed(); this.appsKilled = metrics.getAppsKilled(); - + this.reservedMB = metrics.getReservedMB(); this.availableMB = metrics.getAvailableMB(); this.allocatedMB = metrics.getAllocatedMB(); - + + this.reservedVirtualCores = metrics.getReservedVirtualCores(); + this.availableVirtualCores = metrics.getAvailableVirtualCores(); + this.allocatedVirtualCores = metrics.getAllocatedVirtualCores(); + this.containersAllocated = metrics.getAllocatedContainers(); this.containersPending = metrics.getPendingContainers(); this.containersReserved = metrics.getReservedContainers(); - + this.totalMB = availableMB + allocatedMB; + this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; this.activeNodes = clusterMetrics.getNumActiveNMs(); this.lostNodes = clusterMetrics.getNumLostNMs(); this.unhealthyNodes = clusterMetrics.getUnhealthyNMs(); @@ -123,6 +133,18 @@ public class ClusterMetricsInfo { return this.allocatedMB; } + public long getReservedVirtualCores() { + return this.reservedVirtualCores; + } + + public long getAvailableVirtualCores() { + return this.availableVirtualCores; + } + + public long getAllocatedVirtualCores() { + return this.allocatedVirtualCores; + } + public int getContainersAllocated() { return this.containersAllocated; } @@ -134,15 +156,19 @@ public class ClusterMetricsInfo { public int getPendingContainers() { return this.containersPending; } - + public long getTotalMB() { return this.totalMB; } + public long getTotalVirtualCores() { + return this.totalVirtualCores; + } + public int getTotalNodes() { return this.totalNodes; } - + public int getActiveNodes() { return this.activeNodes; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java Tue Aug 19 23:49:39 2014 @@ -46,8 +46,7 @@ public class FairSchedulerInfo extends S } public int getAppFairShare(ApplicationAttemptId appAttemptId) { - return scheduler.getSchedulerApp(appAttemptId). - getAppSchedulable().getFairShare().getMemory(); + return scheduler.getSchedulerApp(appAttemptId).getFairShare().getMemory(); } public FairSchedulerQueueInfo getRootQueueInfo() { Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java Tue Aug 19 23:49:39 2014 @@ -24,7 +24,8 @@ import javax.xml.bind.annotation.XmlAcce import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .FSAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; @@ -39,9 +40,9 @@ public class FairSchedulerLeafQueueInfo public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) { super(queue, scheduler); - Collection<AppSchedulable> apps = queue.getRunnableAppSchedulables(); - for (AppSchedulable app : apps) { - if (app.getApp().isPending()) { + Collection<FSAppAttempt> apps = queue.getRunnableAppSchedulables(); + for (FSAppAttempt app : apps) { + if (app.isPending()) { numPendingApps++; } else { numActiveApps++; Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java Tue Aug 19 23:49:39 2014 @@ -70,7 +70,7 @@ public class FairSchedulerQueueInfo { queueName = queue.getName(); schedulingPolicy = queue.getPolicy().getName(); - clusterResources = new ResourceInfo(scheduler.getClusterCapacity()); + clusterResources = new ResourceInfo(scheduler.getClusterResource()); usedResources = new ResourceInfo(queue.getResourceUsage()); fractionMemUsed = (float)usedResources.getMemory() / @@ -81,7 +81,7 @@ public class FairSchedulerQueueInfo { maxResources = new ResourceInfo(queue.getMaxShare()); maxResources = new ResourceInfo( Resources.componentwiseMin(queue.getMaxShare(), - scheduler.getClusterCapacity())); + scheduler.getClusterResource())); fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory(); fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory(); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java Tue Aug 19 23:49:39 2014 @@ -43,6 +43,8 @@ public class NodeInfo { protected int numContainers; protected long usedMemoryMB; protected long availMemoryMB; + protected long usedVirtualCores; + protected long availableVirtualCores; public NodeInfo() { } // JAXB needs this @@ -57,6 +59,8 @@ public class NodeInfo { this.numContainers = report.getNumContainers(); this.usedMemoryMB = report.getUsedResource().getMemory(); this.availMemoryMB = report.getAvailableResource().getMemory(); + this.usedVirtualCores = report.getUsedResource().getVirtualCores(); + this.availableVirtualCores = report.getAvailableResource().getVirtualCores(); } this.id = id.toString(); this.rack = ni.getRackName(); @@ -83,7 +87,7 @@ public class NodeInfo { public String getNodeHTTPAddress() { return this.nodeHTTPAddress; } - + public void setNodeHTTPAddress(String nodeHTTPAddress) { this.nodeHTTPAddress = nodeHTTPAddress; } @@ -112,4 +116,12 @@ public class NodeInfo { return this.availMemoryMB; } + public long getUsedVirtualCores() { + return this.usedVirtualCores; + } + + public long getAvailableVirtualCores() { + return this.availableVirtualCores; + } + } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java Tue Aug 19 23:49:39 2014 @@ -30,7 +30,7 @@ public class ResourceInfo { int memory; int vCores; - public ResourceInfo() { + public ResourceInfo() { } public ResourceInfo(Resource res) { @@ -50,4 +50,12 @@ public class ResourceInfo { public String toString() { return "<memory:" + memory + ", vCores:" + vCores + ">"; } + + public void setMemory(int memory) { + this.memory = memory; + } + + public void setvCores(int vCores) { + this.vCores = vCores; + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java Tue Aug 19 23:49:39 2014 @@ -43,6 +43,9 @@ public class UserMetricsInfo { protected long reservedMB; protected long pendingMB; protected long allocatedMB; + protected long reservedVirtualCores; + protected long pendingVirtualCores; + protected long allocatedVirtualCores; @XmlTransient protected boolean userMetricsAvailable; @@ -59,7 +62,7 @@ public class UserMetricsInfo { if (userMetrics != null) { this.userMetricsAvailable = true; - + this.appsSubmitted = userMetrics.getAppsSubmitted(); this.appsCompleted = metrics.getAppsCompleted(); this.appsPending = metrics.getAppsPending(); @@ -70,10 +73,14 @@ public class UserMetricsInfo { this.runningContainers = userMetrics.getAllocatedContainers(); this.pendingContainers = userMetrics.getPendingContainers(); this.reservedContainers = userMetrics.getReservedContainers(); - + this.reservedMB = userMetrics.getReservedMB(); this.pendingMB = userMetrics.getPendingMB(); this.allocatedMB = userMetrics.getAllocatedMB(); + + this.reservedVirtualCores = userMetrics.getReservedVirtualCores(); + this.pendingVirtualCores = userMetrics.getPendingVirtualCores(); + this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores(); } } @@ -117,6 +124,18 @@ public class UserMetricsInfo { return this.pendingMB; } + public long getReservedVirtualCores() { + return this.reservedVirtualCores; + } + + public long getAllocatedVirtualCores() { + return this.allocatedVirtualCores; + } + + public long getPendingVirtualCores() { + return this.pendingVirtualCores; + } + public int getReservedContainers() { return this.reservedContainers; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java Tue Aug 19 23:49:39 2014 @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.List; import org.junit.Assert; - import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; @@ -34,6 +33,7 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; @@ -49,7 +49,7 @@ public class MockAM { private volatile int responseId = 0; private final ApplicationAttemptId attemptId; - private final RMContext context; + private RMContext context; private ApplicationMasterProtocol amRMProtocol; private final List<ResourceRequest> requests = new ArrayList<ResourceRequest>(); @@ -61,8 +61,10 @@ public class MockAM { this.amRMProtocol = amRMProtocol; this.attemptId = attemptId; } - - void setAMRMProtocol(ApplicationMasterProtocol amRMProtocol) { + + public void setAMRMProtocol(ApplicationMasterProtocol amRMProtocol, + RMContext context) { + this.context = context; this.amRMProtocol = amRMProtocol; } @@ -251,4 +253,22 @@ public class MockAM { public ApplicationAttemptId getApplicationAttemptId() { return this.attemptId; } + + public List<Container> allocateAndWaitForContainers(int nContainer, + int memory, MockNM nm) throws Exception { + // AM request for containers + allocate("ANY", memory, nContainer, null); + // kick the scheduler + nm.nodeHeartbeat(true); + List<Container> conts = + allocate(new ArrayList<ResourceRequest>(), null) + .getAllocatedContainers(); + while (conts.size() < nContainer) { + nm.nodeHeartbeat(true); + conts.addAll(allocate(new ArrayList<ResourceRequest>(), + new ArrayList<ContainerId>()).getAllocatedContainers()); + Thread.sleep(500); + } + return conts; + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java Tue Aug 19 23:49:39 2014 @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -32,7 +31,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; @@ -88,7 +87,7 @@ public class MockNM { return httpPort; } - void setResourceTrackerService(ResourceTrackerService resourceTracker) { + public void setResourceTrackerService(ResourceTrackerService resourceTracker) { this.resourceTracker = resourceTracker; } @@ -101,19 +100,26 @@ public class MockNM { } public RegisterNodeManagerResponse registerNode() throws Exception { - return registerNode(null); + return registerNode(null, null); + } + + public RegisterNodeManagerResponse registerNode( + List<ApplicationId> runningApplications) throws Exception { + return registerNode(null, runningApplications); } public RegisterNodeManagerResponse registerNode( - List<ContainerStatus> containerStatus) throws Exception{ + List<NMContainerStatus> containerReports, + List<ApplicationId> runningApplications) throws Exception { RegisterNodeManagerRequest req = Records.newRecord( RegisterNodeManagerRequest.class); req.setNodeId(nodeId); req.setHttpPort(httpPort); Resource resource = BuilderUtils.newResource(memory, vCores); req.setResource(resource); - req.setContainerStatuses(containerStatus); + req.setContainerStatuses(containerReports); req.setNMVersion(version); + req.setRunningApplications(runningApplications); RegisterNodeManagerResponse registrationResponse = resourceTracker.registerNodeManager(req); this.currentContainerTokenMasterKey = @@ -185,4 +191,11 @@ public class MockNM { return heartbeatResponse; } + public int getMemory() { + return memory; + } + + public int getvCores() { + return vCores; + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Tue Aug 19 23:49:39 2014 @@ -21,14 +21,14 @@ package org.apache.hadoop.yarn.server.re import java.io.IOException; import java.nio.ByteBuffer; import java.security.PrivilegedAction; +import java.util.List; import java.util.Map; -import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -47,12 +47,15 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -61,6 +64,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -69,13 +73,19 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStartedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import org.junit.Assert; @SuppressWarnings("unchecked") public class MockRM extends ResourceManager { @@ -144,11 +154,39 @@ public class MockRM extends ResourceMana } } + public void waitForContainerToComplete(RMAppAttempt attempt, + NMContainerStatus completedContainer) throws InterruptedException { + while (true) { + List<ContainerStatus> containers = attempt.getJustFinishedContainers(); + System.out.println("Received completed containers " + containers); + for (ContainerStatus container : containers) { + if (container.getContainerId().equals( + completedContainer.getContainerId())) { + return; + } + } + Thread.sleep(200); + } + } + + public MockAM waitForNewAMToLaunchAndRegister(ApplicationId appId, int attemptSize, + MockNM nm) throws Exception { + RMApp app = getRMContext().getRMApps().get(appId); + Assert.assertNotNull(app); + while (app.getAppAttempts().size() != attemptSize) { + System.out.println("Application " + appId + + " is waiting for AM to restart. Current has " + + app.getAppAttempts().size() + " attempts."); + Thread.sleep(200); + } + return launchAndRegisterAM(app, this, nm); + } + public void waitForState(MockNM nm, ContainerId containerId, RMContainerState containerState) throws Exception { RMContainer container = getResourceScheduler().getRMContainer(containerId); int timeoutSecs = 0; - while(container == null && timeoutSecs++ < 20) { + while(container == null && timeoutSecs++ < 100) { nm.nodeHeartbeat(true); container = getResourceScheduler().getRMContainer(containerId); System.out.println("Waiting for container " + containerId + " to be allocated."); @@ -329,11 +367,20 @@ public class MockRM extends ResourceMana nm.registerNode(); return nm; } + + public MockNM registerNode(String nodeIdStr, int memory, int vCores, + List<ApplicationId> runningApplications) throws Exception { + MockNM nm = + new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService(), + YarnVersionInfo.getVersion()); + nm.registerNode(runningApplications); + return nm; + } public void sendNodeStarted(MockNM nm) throws Exception { RMNodeImpl node = (RMNodeImpl) getRMContext().getRMNodes().get( nm.getNodeId()); - node.handle(new RMNodeEvent(nm.getNodeId(), RMNodeEventType.STARTED)); + node.handle(new RMNodeStartedEvent(nm.getNodeId(), null, null)); } public void sendNodeLost(MockNM nm) throws Exception { @@ -368,6 +415,13 @@ public class MockRM extends ResourceMana throws Exception { MockAM am = new MockAM(getRMContext(), masterService, appAttemptId); am.waitForState(RMAppAttemptState.ALLOCATED); + //create and set AMRMToken + Token<AMRMTokenIdentifier> amrmToken = + this.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken( + appAttemptId); + ((RMAppAttemptImpl) this.rmContext.getRMApps() + .get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId)) + .setAMRMToken(amrmToken); getRMContext() .getDispatcher() .getEventHandler() @@ -520,6 +574,7 @@ public class MockRM extends ResourceMana throws Exception { rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); RMAppAttempt attempt = app.getCurrentAppAttempt(); + System.out.println("Launch AM " + attempt.getAppAttemptId()); nm.nodeHeartbeat(true); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); @@ -542,4 +597,16 @@ public class MockRM extends ResourceMana .newInstance(appId)); return response.getApplicationReport(); } + + // Explicitly reset queue metrics for testing. + @SuppressWarnings("static-access") + public void clearQueueMetrics(RMApp app) { + ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) getResourceScheduler()) + .getSchedulerApplications().get(app.getApplicationId()).getQueue() + .getMetrics().clearQueueMetrics(); + } + + public RMActiveServices getRMActiveService() { + return activeServices; + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java Tue Aug 19 23:49:39 2014 @@ -59,8 +59,9 @@ public class MockRMWithCustomAMLauncher return containerManager; } @Override - protected Token<AMRMTokenIdentifier> getAMRMToken() { - Token<AMRMTokenIdentifier> amRmToken = super.getAMRMToken(); + protected Token<AMRMTokenIdentifier> createAndSetAMRMToken() { + Token<AMRMTokenIdentifier> amRmToken = + super.createAndSetAMRMToken(); InetSocketAddress serviceAddr = getConfig().getSocketAddr( YarnConfiguration.RM_SCHEDULER_ADDRESS, Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Tue Aug 19 23:49:39 2014 @@ -32,6 +32,7 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.DataInputByteBuffer; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -272,21 +273,62 @@ public class TestAMAuthorization { client.registerApplicationMaster(request); Assert.fail("Should fail with authorization error"); } catch (Exception e) { - // Because there are no tokens, the request should be rejected as the - // server side will assume we are trying simple auth. - String expectedMessage = ""; - if (UserGroupInformation.isSecurityEnabled()) { - expectedMessage = "Client cannot authenticate via:[TOKEN]"; + if (isCause(AccessControlException.class, e)) { + // Because there are no tokens, the request should be rejected as the + // server side will assume we are trying simple auth. + String expectedMessage = ""; + if (UserGroupInformation.isSecurityEnabled()) { + expectedMessage = "Client cannot authenticate via:[TOKEN]"; + } else { + expectedMessage = + "SIMPLE authentication is not enabled. Available:[TOKEN]"; + } + Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); } else { - expectedMessage = - "SIMPLE authentication is not enabled. Available:[TOKEN]"; + throw e; } - Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); } // TODO: Add validation of invalid authorization when there's more data in // the AMRMToken } + + /** + * Identify if an expected throwable included in an exception stack. We use + * this because sometimes, an exception will be wrapped to another exception + * before thrown. Like, + * + * <pre> + * {@code + * void methodA() throws IOException { + * try { + * // something + * } catch (AccessControlException e) { + * // do process + * throw new IOException(e) + * } + * } + * </pre> + * + * So we cannot simply catch AccessControlException by using + * <pre> + * {@code + * try { + * methodA() + * } catch (AccessControlException e) { + * // do something + * } + * </pre> + * + * This method is useful in such cases. + */ + private static boolean isCause( + Class<? extends Throwable> expected, + Throwable e + ) { + return (e != null) + && (expected.isInstance(e) || isCause(expected, e.getCause())); + } private void waitForLaunchedState(RMAppAttempt attempt) throws InterruptedException {
