Author: zjshen
Date: Sun Jul 27 17:59:46 2014
New Revision: 1613823
URL: http://svn.apache.org/r1613823
Log:
YARN-2247. Made RM web services authenticate users via kerberos and delegation
token. Contributed by Varun Vasudev.
svn merge --ignore-ancestry -c 1613821 ../../trunk/
Added:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/
- copied from r1613821,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMAuthenticationHandler.java
- copied unchanged from r1613821,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMAuthenticationHandler.java
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
- copied unchanged from r1613821,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java
- copied unchanged from r1613821,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt?rev=1613823&r1=1613822&r2=1613823&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt Sun Jul
27 17:59:46 2014
@@ -39,6 +39,9 @@ Release 2.5.0 - UNRELEASED
YARN-2233. Implemented ResourceManager web-services to create, renew and
cancel delegation tokens. (Varun Vasudev via vinodkv)
+ YARN-2247. Made RM web services authenticate users via kerberos and
delegation
+ token. (Varun Vasudev via zjshen)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1613823&r1=1613822&r2=1613823&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Sun Jul 27 17:59:46 2014
@@ -263,6 +263,17 @@ public class YarnConfiguration extends C
public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
RM_PREFIX + "webapp.spnego-keytab-file";
+ /**
+ * Flag to enable override of the default kerberos authentication filter with
+ * the RM authentication filter to allow authentication using delegation
+ * tokens(fallback to kerberos if the tokens are missing). Only applicable
+ * when the http authentication type is kerberos.
+ */
+ public static final String RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = RM_PREFIX
+ + "webapp.delegation-token-auth-filter.enabled";
+ public static final boolean DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER =
+ true;
+
/** How long to wait until a container is considered dead.*/
public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS =
RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1613823&r1=1613822&r2=1613823&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Sun Jul 27 17:59:46 2014
@@ -195,6 +195,15 @@
</property>
<property>
+ <description>Flag to enable override of the default kerberos authentication
+ filter with the RM authentication filter to allow authentication using
+ delegation tokens(fallback to kerberos if the tokens are missing). Only
+ applicable when the http authentication type is kerberos.</description>
+
<name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
<description>How long to wait until a node manager is considered
dead.</description>
<name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
<value>600000</value>
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1613823&r1=1613822&r2=1613823&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Sun Jul 27 17:59:46 2014
@@ -32,11 +32,13 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import
org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
@@ -88,8 +90,11 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
+import
org.apache.hadoop.yarn.server.resourcemanager.security.RMAuthenticationHandler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter;
+import
org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxy;
@@ -789,6 +794,62 @@ public class ResourceManager extends Com
}
protected void startWepApp() {
+
+ // Use the customized yarn filter instead of the standard kerberos filter
to
+ // allow users to authenticate using delegation tokens
+ // 3 conditions need to be satisfied -
+ // 1. security is enabled
+ // 2. http auth type is set to kerberos
+ // 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true
+
+ Configuration conf = getConfig();
+ boolean useYarnAuthenticationFilter =
+ conf.getBoolean(
+ YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER);
+ String authPrefix = "hadoop.http.authentication.";
+ String authTypeKey = authPrefix + "type";
+ String initializers = conf.get("hadoop.http.filter.initializers");
+ if (UserGroupInformation.isSecurityEnabled()
+ && useYarnAuthenticationFilter
+ && conf.get(authTypeKey, "").equalsIgnoreCase(
+ KerberosAuthenticationHandler.TYPE)) {
+ LOG.info("Using RM authentication filter(kerberos/delegation-token)"
+ + " for RM webapp authentication");
+ RMAuthenticationHandler
+ .setSecretManager(getClientRMService().rmDTSecretManager);
+ String yarnAuthKey =
+ authPrefix + RMAuthenticationFilter.AUTH_HANDLER_PROPERTY;
+ conf.setStrings(yarnAuthKey, RMAuthenticationHandler.class.getName());
+
+ initializers =
+ initializers == null || initializers.isEmpty() ? "" : ","
+ + initializers;
+ if (!initializers.contains(RMAuthenticationFilterInitializer.class
+ .getName())) {
+ conf.set("hadoop.http.filter.initializers",
+ RMAuthenticationFilterInitializer.class.getName() + initializers);
+ }
+ }
+
+ // if security is not enabled and the default filter initializer has been
+ // set, set the initializer to include the
+ // RMAuthenticationFilterInitializer which in turn will set up the simple
+ // auth filter.
+
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ if (initializers == null || initializers.isEmpty()) {
+ conf.set("hadoop.http.filter.initializers",
+ RMAuthenticationFilterInitializer.class.getName());
+ conf.set(authTypeKey, "simple");
+ } else if (initializers.equals(StaticUserWebFilter.class.getName())) {
+ conf.set("hadoop.http.filter.initializers",
+ RMAuthenticationFilterInitializer.class.getName() + ","
+ + initializers);
+ conf.set(authTypeKey, "simple");
+ }
+ }
+
Builder<ApplicationMasterService> builder =
WebApps
.$for("cluster", ApplicationMasterService.class, masterService,
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1613823&r1=1613822&r2=1613823&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
Sun Jul 27 17:59:46 2014
@@ -55,6 +55,7 @@ import org.apache.commons.codec.binary.B
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
@@ -680,6 +681,11 @@ public class RMWebServices {
throw new AuthorizationException(msg);
}
+ if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+ String msg = "The default static user cannot carry out this operation.";
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ }
+
String userName = callerUGI.getUserName();
RMApp app = null;
try {
@@ -800,6 +806,13 @@ public class RMWebServices {
return callerUGI;
}
+ private boolean isStaticUser(UserGroupInformation callerUGI) {
+ String staticUser =
+ conf.get(CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER,
+ CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER);
+ return staticUser.equals(callerUGI.getUserName());
+ }
+
/**
* Generates a new ApplicationId which is then sent to the client
*
@@ -822,6 +835,10 @@ public class RMWebServices {
throw new AuthorizationException("Unable to obtain user name, "
+ "user not authenticated");
}
+ if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+ String msg = "The default static user cannot carry out this operation.";
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ }
NewApplication appId = createNewApplication();
return Response.status(Status.OK).entity(appId).build();
@@ -859,6 +876,11 @@ public class RMWebServices {
+ "user not authenticated");
}
+ if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+ String msg = "The default static user cannot carry out this operation.";
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ }
+
ApplicationSubmissionContext appContext =
createAppSubmissionContext(newApp);
final SubmitApplicationRequest req =
@@ -975,7 +997,7 @@ public class RMWebServices {
*
* @param newApp
* the information provided by the user
- * @return
+ * @return created context
* @throws BadRequestException
* @throws IOException
*/
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1613823&r1=1613822&r2=1613823&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Sun Jul 27 17:59:46 2014
@@ -2912,3 +2912,24 @@ Accept: application/xml
+---+
No response body.
+
+** Authentication using delegation tokens
+
+ This feature is in the alpha mode and may change in the future.
+
+ You can use delegation tokens to authenticate yourself when using YARN RM
webservices. However, this requires setting the right configurations. The
conditions for this are:
+
+ * Hadoop is setup in secure mode with the authentication type set to
kerberos.
+
+ * Hadoop HTTP authentication is setup with the authentication type set to
kerberos
+
+ Once setup, delegation tokens can be fetched using the web services listed
above and used as shown in an example below:
+
++---+
+ PUT http://<rm http
address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+ Hadoop-YARN-Auth-Delegation-Token:
MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
+ Content-Type: application/json; charset=UTF8
+ {
+ "state":"KILLED"
+ }
++---+
