Repository: sqoop Updated Branches: refs/heads/sqoop2 02786f0e2 -> c89f168d4
SQOOP-1776: Sqoop2: Delegation Token support for Authentication (Richard Zhou via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c89f168d Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c89f168d Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c89f168d Branch: refs/heads/sqoop2 Commit: c89f168d494ee9a73b784dc9bab0d8465ccc97e4 Parents: 02786f0 Author: Abraham Elmahrek <[email protected]> Authored: Sun Dec 28 00:22:52 2014 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Sun Dec 28 00:22:52 2014 -0800 ---------------------------------------------------------------------- client/pom.xml | 3 +- .../sqoop/client/request/ResourceRequest.java | 52 ++++++++++++++++++-- .../org/apache/sqoop/common/MapContext.java | 27 +++++++++- .../sqoop/security/AuthenticationConstants.java | 5 ++ dist/src/main/server/conf/sqoop.properties | 6 ++- pom.xml | 22 ++------- .../sqoop/filter/SqoopAuthenticationFilter.java | 29 +++++++++-- shell/pom.xml | 4 -- test/pom.xml | 21 ++------ .../minicluster/TomcatSqoopMiniCluster.java | 2 + 10 files changed, 121 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/client/pom.xml ---------------------------------------------------------------------- diff --git a/client/pom.xml b/client/pom.xml index b53c9d3..d424e8d 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -57,8 +57,7 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - <scope>provided</scope> + <artifactId>hadoop-common</artifactId> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java index 5f134d5..e55ba36 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java @@ -17,8 +17,13 @@ */ package org.apache.sqoop.client.request; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; import org.apache.log4j.Logger; import org.apache.sqoop.client.ClientError; import org.apache.sqoop.common.SqoopException; @@ -34,6 +39,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; +import java.net.InetSocketAddress; import java.net.URL; import java.util.Locale; @@ -42,6 +48,8 @@ import java.util.Locale; */ public class ResourceRequest { private static final Logger LOG = Logger.getLogger(ResourceRequest.class); + private DelegationTokenAuthenticatedURL.Token authToken; + private String strURL; protected String doHttpRequest(String strURL, String method) { return doHttpRequest(strURL, method, ""); @@ -51,9 +59,10 @@ public class ResourceRequest { DataOutputStream wr = null; BufferedReader reader = null; try { - AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + this.authToken = new DelegationTokenAuthenticatedURL.Token(); + this.strURL = strURL; URL url = new URL(strURL); - HttpURLConnection conn = new AuthenticatedURL().openConnection(url, token); + HttpURLConnection conn = new DelegationTokenAuthenticatedURL().openConnection(url, authToken); conn.setRequestMethod(method); // Sqoop is using JSON for data transfers @@ -166,4 +175,41 @@ public class ResourceRequest { public String delete(String url) { return doHttpRequest(url, HttpMethod.DELETE); } + + public Token<?>[] addDelegationTokens(String renewer, + Credentials credentials) throws IOException { + Token<?>[] tokens = null; + Text dtService = getDelegationTokenService(); + Token<?> token = credentials.getToken(dtService); + if (token == null) { + URL url = new URL(strURL); + DelegationTokenAuthenticatedURL authUrl = + new DelegationTokenAuthenticatedURL(new ConnectionConfigurator() { + @Override + public HttpURLConnection configure(HttpURLConnection conn) throws IOException { + return conn; + } + }); + try { + token = authUrl.getDelegationToken(url, authToken, renewer); + if (token != null) { + credentials.addToken(token.getService(), token); + tokens = new Token<?>[]{token}; + } else { + throw new IOException("Got NULL as delegation token"); + } + } catch (AuthenticationException ex) { + throw new IOException(ex); + } + } + return tokens; + } + + private Text getDelegationTokenService() throws IOException { + URL url = new URL(strURL); + InetSocketAddress addr = new InetSocketAddress(url.getHost(), + url.getPort()); + Text dtService = SecurityUtil.buildTokenService(addr); + return dtService; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/common/src/main/java/org/apache/sqoop/common/MapContext.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/common/MapContext.java b/common/src/main/java/org/apache/sqoop/common/MapContext.java index 2229889..d1f9420 100644 --- a/common/src/main/java/org/apache/sqoop/common/MapContext.java +++ b/common/src/main/java/org/apache/sqoop/common/MapContext.java @@ -19,6 +19,8 @@ package org.apache.sqoop.common; import java.util.HashMap; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * ImmutableContext implementation based on (Hash)Map. @@ -71,7 +73,7 @@ public class MapContext implements ImmutableContext { */ @Override public long getLong(String key, long defaultValue) { - if(!options.containsKey(key)) { + if (!options.containsKey(key)) { return defaultValue; } @@ -85,7 +87,7 @@ public class MapContext implements ImmutableContext { */ @Override public int getInt(String key, int defaultValue) { - if(!options.containsKey(key)) { + if (!options.containsKey(key)) { return defaultValue; } @@ -110,4 +112,25 @@ public class MapContext implements ImmutableContext { return subProps; } + + /** + * get keys matching the the regex + * + * @param regex + * @return Map<String,String> with matching keys + */ + public Map<String, String> getValByRegex(String regex) { + Pattern p = Pattern.compile(regex); + + Map<String, String> result = new HashMap<String, String>(); + Matcher m; + + for (String item : options.keySet()) { + m = p.matcher(item); + if (m.find()) { // match + result.put(item, getString(item)); + } + } + return result; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java index f37e505..ec2f32d 100644 --- a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java +++ b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java @@ -96,6 +96,11 @@ public final class AuthenticationConstants { public static final String AUTHENTICATION_KERBEROS_HTTP_KEYTAB = PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "keytab"; + /** + * The config specifies the token kind in delegation token. + */ + public static final String TOKEN_KIND = "sqoop_token_kind"; + public static enum TYPE {SIMPLE, KERBEROS} private AuthenticationConstants() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/dist/src/main/server/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties index fe4dfb6..2ae1aba 100755 --- a/dist/src/main/server/conf/sqoop.properties +++ b/dist/src/main/server/conf/sqoop.properties @@ -152,4 +152,8 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce #org.apache.sqoop.authentication.kerberos.principal=sqoop/_HOST@NOVALOCAL #org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab #org.apache.sqoop.authentication.kerberos.http.principal=HTTP/_HOST@NOVALOCAL -#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab \ No newline at end of file +#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab +#org.apache.sqoop.authentication.enable.doAs=true +#org.apache.sqoop.authentication.proxyuser.#USER#.users=* +#org.apache.sqoop.authentication.proxyuser.#USER#.groups=* +#org.apache.sqoop.authentication.proxyuser.#USER#.hosts=* \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c7c9d78..ea157f7 100644 --- a/pom.xml +++ b/pom.xml @@ -228,23 +228,6 @@ limitations under the License. <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.2.version}</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.2.version}</version> <scope>provided</scope> @@ -605,6 +588,11 @@ limitations under the License. <artifactId>jackson-databind</artifactId> <version>${jackson.databind.version}</version> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.2.version}</version> + </dependency> </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java index bb66290..2b6ef34 100644 --- a/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java +++ b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java @@ -17,10 +17,15 @@ */ package org.apache.sqoop.filter; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; import org.apache.sqoop.common.MapContext; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.core.SqoopConfiguration; @@ -30,21 +35,22 @@ import org.apache.sqoop.security.AuthenticationError; import javax.servlet.FilterConfig; import javax.servlet.ServletException; import java.io.IOException; +import java.util.Map; import java.util.Properties; -public class SqoopAuthenticationFilter extends AuthenticationFilter { +public class SqoopAuthenticationFilter extends DelegationTokenAuthenticationFilter { @Override protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException { - Properties properties = super.getConfiguration(configPrefix, filterConfig); + Properties properties = new Properties(); MapContext mapContext = SqoopConfiguration.getInstance().getContext(); String type = mapContext.getString( AuthenticationConstants.AUTHENTICATION_TYPE, AuthenticationConstants.TYPE.SIMPLE.name()).trim(); if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.KERBEROS.name())) { - properties.setProperty(AUTH_TYPE, AuthenticationConstants.TYPE.KERBEROS.name().toLowerCase()); + properties.setProperty(AUTH_TYPE, KerberosDelegationTokenAuthenticationHandler.class.getName()); String keytab = mapContext.getString( AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB).trim(); @@ -71,13 +77,26 @@ public class SqoopAuthenticationFilter extends AuthenticationFilter { properties.setProperty(KerberosAuthenticationHandler.PRINCIPAL, hostPrincipal); properties.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab); } else if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.SIMPLE.name())) { - properties.setProperty(AUTH_TYPE, PseudoAuthenticationHandler.class.getName()); + properties.setProperty(AUTH_TYPE, PseudoDelegationTokenAuthenticationHandler.class.getName()); properties.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, mapContext.getString(AuthenticationConstants.AUTHENTICATION_ANONYMOUS, "true").trim()); } else { throw new SqoopException(AuthenticationError.AUTH_0004, type); } + properties.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND, + AuthenticationConstants.TOKEN_KIND); + return properties; } -} + + protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) { + MapContext mapContext = SqoopConfiguration.getInstance().getContext(); + Map<String, String> proxyuserConf = mapContext.getValByRegex("org\\.apache\\.sqoop\\.authentication\\.proxyuser"); + Configuration conf = new Configuration(false); + for (Map.Entry<String, String> entry : proxyuserConf.entrySet()) { + conf.set(entry.getKey().substring("org.apache.sqoop.authentication.proxyuser.".length()), entry.getValue()); + } + return conf; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/shell/pom.xml ---------------------------------------------------------------------- diff --git a/shell/pom.xml b/shell/pom.xml index 2092690..e48c86a 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -70,10 +70,6 @@ limitations under the License. <artifactId>groovy-all</artifactId> <version>1.8.5</version> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - </dependency> </dependencies> <profiles> http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index 38b2279..f74ee3c 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -122,6 +122,11 @@ limitations under the License. <artifactId>commons-compress</artifactId> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + </dependencies> <!-- Add classifier name to the JAR name --> @@ -232,22 +237,6 @@ limitations under the License. <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <exclusions> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </exclusion> - </exclusions> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <scope>provided</scope> </dependency> http://git-wip-us.apache.org/repos/asf/sqoop/blob/c89f168d/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java index 1e6706e..9c099fc 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java @@ -91,6 +91,8 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster { jar.contains("httpcore-") || // Apache Http Core libraries jar.contains("httpclient-") || // Apache Http Client libraries jar.contains("htrace-") || // htrace-core libraries, new added in Hadoop 2.6.0 + jar.contains("zookeeper-") || // zookeeper libraries, new added in Hadoop 2.6.0 + jar.contains("curator-") || // curator libraries, new added in Hadoop 2.6.0 jar.contains("log4j-") || // Log4j jar.contains("slf4j-") || // Slf4j jar.contains("jackson-") || // Jackson
