Author: vinodkv
Date: Wed Jul 2 00:23:49 2014
New Revision: 1607217
URL: http://svn.apache.org/r1607217
Log:
YARN-1713. Added get-new-app and submit-app functionality to RM web services.
Contributed by Varun Vasudev.
svn merge --ignore-ancestry -c 1607216 ../../trunk/
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
- copied unchanged from r1607216,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java
- copied unchanged from r1607216,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java
- copied unchanged from r1607216,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java
- copied unchanged from r1607216,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java
- copied unchanged from r1607216,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1607217&r1=1607216&r2=1607217&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Wed Jul 2
00:23:49 2014
@@ -33,6 +33,9 @@ Release 2.5.0 - UNRELEASED
YARN-2052. Embedded an epoch number in container id to ensure the
uniqueness
of container id after RM restarts. (Tsuyoshi OZAWA via jianhe)
+ YARN-1713. Added get-new-app and submit-app functionality to RM web
services.
+ (Varun Vasudev via vinodkv)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java?rev=1607217&r1=1607216&r2=1607217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
Wed Jul 2 00:23:49 2014
@@ -88,7 +88,7 @@ public abstract class ApplicationSubmiss
int maxAppAttempts, Resource resource, String applicationType) {
return newInstance(applicationId, applicationName, queue, priority,
amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
- resource, null, false);
+ resource, applicationType, false);
}
@Public
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java?rev=1607217&r1=1607216&r2=1607217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
Wed Jul 2 00:23:49 2014
@@ -21,10 +21,12 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
import javax.ws.rs.ext.Provider;
+import javax.xml.bind.UnmarshalException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -87,6 +89,9 @@ public class GenericExceptionHandler imp
s = Response.Status.BAD_REQUEST;
} else if (e instanceof BadRequestException) {
s = Response.Status.BAD_REQUEST;
+ } else if (e instanceof WebApplicationException
+ && e.getCause() instanceof UnmarshalException) {
+ s = Response.Status.BAD_REQUEST;
} else {
LOG.warn("INTERNAL_SERVER_ERROR", e);
s = Response.Status.INTERNAL_SERVER_ERROR;
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1607217&r1=1607216&r2=1607217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
Wed Jul 2 00:23:49 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.security.AccessControlException;
+import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
@@ -36,6 +37,7 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
+import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -47,22 +49,38 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
+import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -81,17 +99,22 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
@@ -758,4 +781,256 @@ public class RMWebServices {
return callerUGI;
}
+
+ /**
+ * Generates a new ApplicationId which is then sent to the client
+ *
+ * @param hsr
+ * the servlet request
+ * @return Response containing the app id and the maximum resource
+ * capabilities
+ * @throws AuthorizationException
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @POST
+ @Path("/apps/new-application")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response createNewApplication(@Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException {
+ init();
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
+ if (callerUGI == null) {
+ throw new AuthorizationException("Unable to obtain user name, "
+ + "user not authenticated");
+ }
+
+ NewApplication appId = createNewApplication();
+ return Response.status(Status.OK).entity(appId).build();
+
+ }
+
+ // reuse the code in ClientRMService to create new app
+ // get the new app id and submit app
+ // set location header with new app location
+ /**
+ * Function to submit an app to the RM
+ *
+ * @param newApp
+ * structure containing information to construct the
+ * ApplicationSubmissionContext
+ * @param hsr
+ * the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @POST
+ @Path("/apps")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response submitApplication(ApplicationSubmissionContextInfo newApp,
+ @Context HttpServletRequest hsr) throws AuthorizationException,
+ IOException, InterruptedException {
+
+ init();
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
+ if (callerUGI == null) {
+ throw new AuthorizationException("Unable to obtain user name, "
+ + "user not authenticated");
+ }
+
+ ApplicationSubmissionContext appContext =
+ createAppSubmissionContext(newApp);
+ final SubmitApplicationRequest req =
+ SubmitApplicationRequest.newInstance(appContext);
+
+ try {
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<SubmitApplicationResponse>() {
+ @Override
+ public SubmitApplicationResponse run() throws IOException,
+ YarnException {
+ return rm.getClientRMService().submitApplication(req);
+ }
+ });
+ } catch (UndeclaredThrowableException ue) {
+ if (ue.getCause() instanceof YarnException) {
+ throw new BadRequestException(ue.getCause().getMessage());
+ }
+ LOG.info("Submit app request failed", ue);
+ throw ue;
+ }
+
+ String url = hsr.getRequestURL() + "/" + newApp.getApplicationId();
+ return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, url)
+ .build();
+ }
+
+ /**
+ * Function that actually creates the ApplicationId by calling the
+ * ClientRMService
+ *
+ * @return returns structure containing the app-id and maximum resource
+ * capabilities
+ */
+ private NewApplication createNewApplication() {
+ GetNewApplicationRequest req =
+ recordFactory.newRecordInstance(GetNewApplicationRequest.class);
+ GetNewApplicationResponse resp;
+ try {
+ resp = rm.getClientRMService().getNewApplication(req);
+ } catch (YarnException e) {
+ String msg = "Unable to create new app from RM web service";
+ LOG.error(msg, e);
+ throw new YarnRuntimeException(msg, e);
+ }
+ NewApplication appId =
+ new NewApplication(resp.getApplicationId().toString(), new
ResourceInfo(
+ resp.getMaximumResourceCapability()));
+ return appId;
+ }
+
+ /**
+ * Create the actual ApplicationSubmissionContext to be submitted to the RM
+ * from the information provided by the user.
+ *
+ * @param newApp
+ * the information provided by the user
+ * @return returns the constructed ApplicationSubmissionContext
+ * @throws IOException
+ */
+ protected ApplicationSubmissionContext createAppSubmissionContext(
+ ApplicationSubmissionContextInfo newApp) throws IOException {
+
+ // create local resources and app submission context
+
+ ApplicationId appid;
+ String error =
+ "Could not parse application id " + newApp.getApplicationId();
+ try {
+ appid =
+ ConverterUtils.toApplicationId(recordFactory,
+ newApp.getApplicationId());
+ } catch (Exception e) {
+ throw new BadRequestException(error);
+ }
+ ApplicationSubmissionContext appContext =
+ ApplicationSubmissionContext.newInstance(appid,
+ newApp.getApplicationName(), newApp.getQueue(),
+ Priority.newInstance(newApp.getPriority()),
+ createContainerLaunchContext(newApp), newApp.getUnmanagedAM(),
+ newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
+ createAppSubmissionContextResource(newApp),
+ newApp.getApplicationType(),
+ newApp.getKeepContainersAcrossApplicationAttempts());
+ appContext.setApplicationTags(newApp.getApplicationTags());
+
+ return appContext;
+ }
+
+ protected Resource createAppSubmissionContextResource(
+ ApplicationSubmissionContextInfo newApp) throws BadRequestException {
+ if (newApp.getResource().getvCores() > rm.getConfig().getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) {
+ String msg = "Requested more cores than configured max";
+ throw new BadRequestException(msg);
+ }
+ if (newApp.getResource().getMemory() > rm.getConfig().getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) {
+ String msg = "Requested more memory than configured max";
+ throw new BadRequestException(msg);
+ }
+ Resource r =
+ Resource.newInstance(newApp.getResource().getMemory(), newApp
+ .getResource().getvCores());
+ return r;
+ }
+
+ /**
+ * Create the ContainerLaunchContext required for the
+ * ApplicationSubmissionContext. This function takes the user information and
+ * generates the ByteBuffer structures required by the ContainerLaunchContext
+ *
+ * @param newApp
+ * the information provided by the user
+ * @return
+ * @throws BadRequestException
+ * @throws IOException
+ */
+ protected ContainerLaunchContext createContainerLaunchContext(
+ ApplicationSubmissionContextInfo newApp) throws BadRequestException,
IOException {
+
+ // create container launch context
+
+ HashMap<String, ByteBuffer> hmap = new HashMap<String, ByteBuffer>();
+ for (Map.Entry<String, String> entry : newApp
+ .getContainerLaunchContextInfo().getAuxillaryServiceData().entrySet()) {
+ if (entry.getValue().isEmpty() == false) {
+ Base64 decoder = new Base64(0, null, true);
+ byte[] data = decoder.decode(entry.getValue());
+ hmap.put(entry.getKey(), ByteBuffer.wrap(data));
+ }
+ }
+
+ HashMap<String, LocalResource> hlr = new HashMap<String, LocalResource>();
+ for (Map.Entry<String, LocalResourceInfo> entry : newApp
+ .getContainerLaunchContextInfo().getResources().entrySet()) {
+ LocalResourceInfo l = entry.getValue();
+ LocalResource lr =
+ LocalResource.newInstance(
+ ConverterUtils.getYarnUrlFromURI(l.getUrl()), l.getType(),
+ l.getVisibility(), l.getSize(), l.getTimestamp());
+ hlr.put(entry.getKey(), lr);
+ }
+
+ DataOutputBuffer out = new DataOutputBuffer();
+ Credentials cs =
+ createCredentials(newApp.getContainerLaunchContextInfo()
+ .getCredentials());
+ cs.writeTokenStorageToStream(out);
+ ByteBuffer tokens = ByteBuffer.wrap(out.getData());
+
+ ContainerLaunchContext ctx =
+ ContainerLaunchContext.newInstance(hlr, newApp
+ .getContainerLaunchContextInfo().getEnvironment(), newApp
+ .getContainerLaunchContextInfo().getCommands(), hmap, tokens, newApp
+ .getContainerLaunchContextInfo().getAcls());
+
+ return ctx;
+ }
+
+ /**
+ * Generate a Credentials object from the information in the CredentialsInfo
+ * object.
+ *
+ * @param credentials
+ * the CredentialsInfo provided by the user.
+ * @return
+ */
+ private Credentials createCredentials(CredentialsInfo credentials) {
+ Credentials ret = new Credentials();
+ try {
+ for (Map.Entry<String, String> entry :
credentials.getTokens().entrySet()) {
+ Text alias = new Text(entry.getKey());
+ Token<TokenIdentifier> token = new Token<TokenIdentifier>();
+ token.decodeFromUrlString(entry.getValue());
+ ret.addToken(alias, token);
+ }
+ for (Map.Entry<String, String> entry :
credentials.getTokens().entrySet()) {
+ Text alias = new Text(entry.getKey());
+ Base64 decoder = new Base64(0, null, true);
+ byte[] secret = decoder.decode(entry.getValue());
+ ret.addSecretKey(alias, secret);
+ }
+ } catch (IOException ie) {
+ throw new BadRequestException(
+ "Could not parse credentials data; exception message = "
+ + ie.getMessage());
+ }
+ return ret;
+ }
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java?rev=1607217&r1=1607216&r2=1607217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
Wed Jul 2 00:23:49 2014
@@ -30,7 +30,7 @@ public class ResourceInfo {
int memory;
int vCores;
- public ResourceInfo() {
+ public ResourceInfo() {
}
public ResourceInfo(Resource res) {
@@ -50,4 +50,12 @@ public class ResourceInfo {
public String toString() {
return "<memory:" + memory + ", vCores:" + vCores + ">";
}
+
+ public void setMemory(int memory) {
+ this.memory = memory;
+ }
+
+ public void setvCores(int vCores) {
+ this.vCores = vCores;
+ }
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java?rev=1607217&r1=1607216&r2=1607217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
Wed Jul 2 00:23:49 2014
@@ -25,10 +25,17 @@ import static org.junit.Assume.assumeTru
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
+import java.net.URI;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
@@ -38,9 +45,15 @@ import javax.xml.parsers.DocumentBuilder
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import
org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -54,7 +67,11 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
@@ -80,6 +97,7 @@ import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.filter.LoggingFilter;
import com.sun.jersey.api.json.JSONJAXBContext;
import com.sun.jersey.api.json.JSONMarshaller;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
@@ -461,11 +479,7 @@ public class TestRMWebServicesAppsModifi
.constructWebResource("apps", app.getApplicationId().toString(),
"state").accept(mediaType)
.entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class);
- if (!isAuthenticationEnabled()) {
- assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
- } else {
- assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
- }
+ validateResponseStatus(response, Status.FORBIDDEN);
}
rm.stop();
return;
@@ -502,4 +516,348 @@ public class TestRMWebServicesAppsModifi
}
super.tearDown();
}
+
+ /**
+ * Helper function to wrap frequently used code. It checks the response
status
+ * and checks if it UNAUTHORIZED if we are running with authorization turned
+ * off or the param passed if we are running with authorization turned on.
+ *
+ * @param response
+ * the ClientResponse object to be checked
+ * @param expectedAuthorizedMode
+ * the expected Status in authorized mode.
+ */
+ public void validateResponseStatus(ClientResponse response,
+ Status expectedAuthorizedMode) {
+ validateResponseStatus(response, Status.UNAUTHORIZED,
+ expectedAuthorizedMode);
+ }
+
+ /**
+ * Helper function to wrap frequently used code. It checks the response
status
+ * and checks if it is the param expectedUnauthorizedMode if we are running
+ * with authorization turned off or the param expectedAuthorizedMode passed
if
+ * we are running with authorization turned on.
+ *
+ * @param response
+ * the ClientResponse object to be checked
+ * @param expectedUnauthorizedMode
+ * the expected Status in unauthorized mode.
+ * @param expectedAuthorizedMode
+ * the expected Status in authorized mode.
+ */
+ public void validateResponseStatus(ClientResponse response,
+ Status expectedUnauthorizedMode, Status expectedAuthorizedMode) {
+ if (!isAuthenticationEnabled()) {
+ assertEquals(expectedUnauthorizedMode,
response.getClientResponseStatus());
+ } else {
+ assertEquals(expectedAuthorizedMode, response.getClientResponseStatus());
+ }
+ }
+
+ // Simple test - just post to /apps/id and validate the response
+ @Test
+ public void testGetNewApplication() throws Exception {
+ // client().addFilter(new LoggingFilter(System.out));
+ rm.start();
+ String mediaTypes[] =
+ { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+ for (String acceptMedia : mediaTypes) {
+ testGetNewApplication(acceptMedia);
+ }
+ rm.stop();
+ return;
+ }
+
+ protected String testGetNewApplication(String mediaType) throws
JSONException,
+ ParserConfigurationException, IOException, SAXException {
+ ClientResponse response =
+ this.constructWebResource("apps", "new-application").accept(mediaType)
+ .post(ClientResponse.class);
+ validateResponseStatus(response, Status.OK);
+ if (!isAuthenticationEnabled()) {
+ return "";
+ }
+ return validateGetNewApplicationResponse(response);
+ }
+
+ protected String validateGetNewApplicationResponse(ClientResponse resp)
+ throws JSONException, ParserConfigurationException, IOException,
+ SAXException {
+ String ret = "";
+ if (resp.getType().equals(MediaType.APPLICATION_JSON_TYPE)) {
+ JSONObject json = resp.getEntity(JSONObject.class);
+ ret = validateGetNewApplicationJsonResponse(json);
+ } else if (resp.getType().equals(MediaType.APPLICATION_XML_TYPE)) {
+ String xml = resp.getEntity(String.class);
+ ret = validateGetNewApplicationXMLResponse(xml);
+ } else {
+ // we should not be here
+ assertTrue(false);
+ }
+ return ret;
+ }
+
+ protected String validateGetNewApplicationJsonResponse(JSONObject json)
+ throws JSONException {
+ String appId = json.getString("application-id");
+ assertTrue(appId.isEmpty() == false);
+ JSONObject maxResources =
json.getJSONObject("maximum-resource-capability");
+ long memory = maxResources.getLong("memory");
+ long vCores = maxResources.getLong("vCores");
+ assertTrue(memory != 0);
+ assertTrue(vCores != 0);
+ return appId;
+ }
+
+ protected String validateGetNewApplicationXMLResponse(String response)
+ throws ParserConfigurationException, IOException, SAXException {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(response));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("NewApplication");
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+ Element element = (Element) nodes.item(0);
+ String appId = WebServicesTestUtils.getXmlString(element,
"application-id");
+ assertTrue(appId.isEmpty() == false);
+ NodeList maxResourceNodes =
+ element.getElementsByTagName("maximum-resource-capability");
+ assertEquals(1, maxResourceNodes.getLength());
+ Element maxResourceCapability = (Element) maxResourceNodes.item(0);
+ long memory =
+ WebServicesTestUtils.getXmlLong(maxResourceCapability, "memory");
+ long vCores =
+ WebServicesTestUtils.getXmlLong(maxResourceCapability, "vCores");
+ assertTrue(memory != 0);
+ assertTrue(vCores != 0);
+ return appId;
+ }
+
+ // Test to validate the process of submitting apps - test for appropriate
+ // errors as well
+ @Test
+ public void testGetNewApplicationAndSubmit() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ amNodeManager.nodeHeartbeat(true);
+ String mediaTypes[] =
+ { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+ for (String acceptMedia : mediaTypes) {
+ for (String contentMedia : mediaTypes) {
+ testAppSubmit(acceptMedia, contentMedia);
+ testAppSubmitErrors(acceptMedia, contentMedia);
+ }
+ }
+ rm.stop();
+ return;
+ }
+
+ public void testAppSubmit(String acceptMedia, String contentMedia)
+ throws Exception {
+
+ // create a test app and submit it via rest(after getting an app-id) then
+ // get the app details from the rmcontext and check that everything matches
+
+ // client().addFilter(new LoggingFilter(System.out));
+ String lrKey = "example";
+ String queueName = "testqueue";
+ String appName = "test";
+ String appType = "test-type";
+ String urlPath = "apps";
+ String appId = testGetNewApplication(acceptMedia);
+ List<String> commands = new ArrayList<String>();
+ commands.add("/bin/sleep 5");
+ HashMap<String, String> environment = new HashMap<String, String>();
+ environment.put("APP_VAR", "ENV_SETTING");
+ HashMap<ApplicationAccessType, String> acls =
+ new HashMap<ApplicationAccessType, String>();
+ acls.put(ApplicationAccessType.MODIFY_APP, "testuser1, testuser2");
+ acls.put(ApplicationAccessType.VIEW_APP, "testuser3, testuser4");
+ Set<String> tags = new HashSet<String>();
+ tags.add("tag1");
+ tags.add("tag 2");
+ CredentialsInfo credentials = new CredentialsInfo();
+ HashMap<String, String> tokens = new HashMap<String, String>();
+ HashMap<String, String> secrets = new HashMap<String, String>();
+ secrets.put("secret1",
Base64.encodeBase64URLSafeString("secret1".getBytes("UTF8")));
+ credentials.setSecrets(secrets);
+ credentials.setTokens(tokens);
+ ApplicationSubmissionContextInfo appInfo = new
ApplicationSubmissionContextInfo();
+ appInfo.setApplicationId(appId);
+ appInfo.setApplicationName(appName);
+ appInfo.setPriority(3);
+ appInfo.setMaxAppAttempts(2);
+ appInfo.setQueue(queueName);
+ appInfo.setApplicationType(appType);
+ HashMap<String, LocalResourceInfo> lr =
+ new HashMap<String, LocalResourceInfo>();
+ LocalResourceInfo y = new LocalResourceInfo();
+ y.setUrl(new URI("http://www.test.com/file.txt";));
+ y.setSize(100);
+ y.setTimestamp(System.currentTimeMillis());
+ y.setType(LocalResourceType.FILE);
+ y.setVisibility(LocalResourceVisibility.APPLICATION);
+ lr.put(lrKey, y);
+ appInfo.getContainerLaunchContextInfo().setResources(lr);
+ appInfo.getContainerLaunchContextInfo().setCommands(commands);
+ appInfo.getContainerLaunchContextInfo().setEnvironment(environment);
+ appInfo.getContainerLaunchContextInfo().setAcls(acls);
+ appInfo.getContainerLaunchContextInfo().getAuxillaryServiceData()
+ .put("test",
Base64.encodeBase64URLSafeString("value12".getBytes("UTF8")));
+ appInfo.getContainerLaunchContextInfo().setCredentials(credentials);
+ appInfo.getResource().setMemory(1024);
+ appInfo.getResource().setvCores(1);
+ appInfo.setApplicationTags(tags);
+
+ ClientResponse response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+
+ if (this.isAuthenticationEnabled() == false) {
+ assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
+ return;
+ }
+ assertEquals(Status.ACCEPTED, response.getClientResponseStatus());
+ assertTrue(response.getHeaders().getFirst(HttpHeaders.LOCATION).isEmpty()
== false);
+ String locURL = response.getHeaders().getFirst(HttpHeaders.LOCATION);
+ assertTrue(locURL.indexOf("/apps/application") != -1);
+ appId = locURL.substring(locURL.indexOf("/apps/") + "/apps/".length());
+
+ WebResource res = resource().uri(new URI(locURL));
+ res = res.queryParam("user.name", webserviceUserName);
+ response = res.get(ClientResponse.class);
+ assertEquals(Status.OK, response.getClientResponseStatus());
+
+ RMApp app =
+ rm.getRMContext().getRMApps()
+ .get(ConverterUtils.toApplicationId(appId));
+ assertEquals(appName, app.getName());
+ assertEquals(webserviceUserName, app.getUser());
+ assertEquals(2, app.getMaxAppAttempts());
+ assertEquals(queueName, app.getQueue());
+ assertEquals(appType, app.getApplicationType());
+ assertEquals(tags, app.getApplicationTags());
+ ContainerLaunchContext ctx =
+ app.getApplicationSubmissionContext().getAMContainerSpec();
+ assertEquals(commands, ctx.getCommands());
+ assertEquals(environment, ctx.getEnvironment());
+ assertEquals(acls, ctx.getApplicationACLs());
+ Map<String, LocalResource> appLRs = ctx.getLocalResources();
+ assertTrue(appLRs.containsKey(lrKey));
+ LocalResource exampleLR = appLRs.get(lrKey);
+ assertEquals(ConverterUtils.getYarnUrlFromURI(y.getUrl()),
+ exampleLR.getResource());
+ assertEquals(y.getSize(), exampleLR.getSize());
+ assertEquals(y.getTimestamp(), exampleLR.getTimestamp());
+ assertEquals(y.getType(), exampleLR.getType());
+ assertEquals(y.getPattern(), exampleLR.getPattern());
+ assertEquals(y.getVisibility(), exampleLR.getVisibility());
+
+ response =
+ this.constructWebResource("apps", appId).accept(acceptMedia)
+ .get(ClientResponse.class);
+ assertEquals(Status.OK, response.getClientResponseStatus());
+ return;
+ }
+
+ public void testAppSubmitErrors(String acceptMedia, String contentMedia)
+ throws Exception {
+
+ // submit a bunch of bad requests(correct format but bad values) via the
+ // REST API and make sure we get the right error response codes
+
+ String urlPath = "apps";
+ String appId = "";
+ ApplicationSubmissionContextInfo appInfo = new
ApplicationSubmissionContextInfo();
+ ClientResponse response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ appId = "random";
+ appInfo.setApplicationId(appId);
+ response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ appId = "random_junk";
+ appInfo.setApplicationId(appId);
+ response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ // bad resource info
+ appInfo.getResource().setMemory(
+ rm.getConfig().getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB) + 1);
+ appInfo.getResource().setvCores(1);
+ response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ appInfo.getResource().setvCores(
+ rm.getConfig().getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES) + 1);
+ appInfo.getResource().setMemory(CONTAINER_MB);
+ response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ return;
+ }
+
+ @Test
+ public void testAppSubmitBadJsonAndXML() throws Exception {
+
+ // submit a bunch of bad XML and JSON via the
+ // REST API and make sure we get error response codes
+
+ String urlPath = "apps";
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ amNodeManager.nodeHeartbeat(true);
+
+ ApplicationSubmissionContextInfo appInfo = new
ApplicationSubmissionContextInfo();
+ appInfo.setApplicationName("test");
+ appInfo.setPriority(3);
+ appInfo.setMaxAppAttempts(2);
+ appInfo.setQueue("testqueue");
+ appInfo.setApplicationType("test-type");
+ HashMap<String, LocalResourceInfo> lr =
+ new HashMap<String, LocalResourceInfo>();
+ LocalResourceInfo y = new LocalResourceInfo();
+ y.setUrl(new URI("http://www.test.com/file.txt";));
+ y.setSize(100);
+ y.setTimestamp(System.currentTimeMillis());
+ y.setType(LocalResourceType.FILE);
+ y.setVisibility(LocalResourceVisibility.APPLICATION);
+ lr.put("example", y);
+ appInfo.getContainerLaunchContextInfo().setResources(lr);
+ appInfo.getResource().setMemory(1024);
+ appInfo.getResource().setvCores(1);
+
+ String body =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" "
+ + "standalone=\"yes\"?><blah/>";
+ ClientResponse response =
+ this.constructWebResource(urlPath).accept(MediaType.APPLICATION_XML)
+ .entity(body, MediaType.APPLICATION_XML).post(ClientResponse.class);
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ body = "{\"a\" : \"b\"}";
+ response =
+ this.constructWebResource(urlPath).accept(MediaType.APPLICATION_XML)
+ .entity(body, MediaType.APPLICATION_JSON).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+ rm.stop();
+ }
+
}
