SQOOP-1527: Sqoop2: Kerberos support (SPNEGO) in communication between server and client
(Richard Zhou via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/1558d940 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/1558d940 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/1558d940 Branch: refs/heads/cdh5-1.99.4 Commit: 1558d940b880d95145943fd67d38bc316a781810 Parents: dc226c4 Author: Abraham Elmahrek <[email protected]> Authored: Sun Nov 16 20:54:13 2014 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Wed Nov 19 13:51:46 2014 -0800 ---------------------------------------------------------------------- client/pom.xml | 11 +- .../sqoop/client/request/ResourceRequest.java | 173 ++++++++++++------- .../sqoop/security/AuthenticationConstants.java | 21 +++ .../sqoop/security/AuthenticationError.java | 8 +- dist/src/main/server/conf/sqoop.properties | 4 +- server/pom.xml | 49 ++++++ .../sqoop/filter/SqoopAuthenticationFilter.java | 80 +++++++++ server/src/main/webapp/WEB-INF/web.xml | 11 ++ shell/pom.xml | 9 +- 9 files changed, 292 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/client/pom.xml ---------------------------------------------------------------------- diff --git a/client/pom.xml b/client/pom.xml index bbc436e..4f7aabd 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -51,9 +51,14 @@ limitations under the License. <artifactId>sqoop-common</artifactId> </dependency> <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - <version>1.11</version> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + <version>2.0.1</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + <scope>provided</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java index c84a83e..028444c 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java @@ -17,15 +17,9 @@ */ package org.apache.sqoop.client.request; -import javax.ws.rs.core.MediaType; - -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientRequest; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.WebResource.Builder; -import com.sun.jersey.api.client.filter.ClientFilter; - +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.log4j.Logger; import org.apache.sqoop.client.ClientError; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.common.SqoopProtocolConstants; @@ -33,81 +27,132 @@ import org.apache.sqoop.json.ThrowableBean; import org.json.simple.JSONObject; import org.json.simple.JSONValue; +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.MediaType; +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; import java.util.Locale; + /** * Represents the sqoop REST resource requests - * */ -public class ResourceRequest -{ - private static ServerExceptionFilter serverExceptionFilter; +public class ResourceRequest { + private static final Logger LOG = Logger.getLogger(ResourceRequest.class); - static { - serverExceptionFilter = new ServerExceptionFilter(); + protected String doHttpRequest(String strURL, String method) { + return doHttpRequest(strURL, method, ""); } - protected Builder getBuilder(String url) { - Client client = Client.create(); - WebResource resource = client.resource(url); + protected String doHttpRequest(String strURL, String method, String data) { + DataOutputStream wr = null; + BufferedReader reader = null; + try { + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + URL url = new URL(strURL); + HttpURLConnection conn = new AuthenticatedURL().openConnection(url, token); + + conn.setRequestMethod(method); +// Provide name of user executing request + conn.setRequestProperty(SqoopProtocolConstants.HEADER_SQOOP_USERNAME, System.getProperty("user.name")); +// Sqoop is using JSON for data transfers + conn.setRequestProperty("Accept", MediaType.APPLICATION_JSON); +// Transfer client locale to return client specific data + conn.setRequestProperty("Accept-Language", Locale.getDefault().toString()); + if (method.equalsIgnoreCase(HttpMethod.PUT) || method.equalsIgnoreCase(HttpMethod.POST)) { + conn.setDoOutput(true); + data = data == null ? "" : data; + conn.setRequestProperty("Content-Length", Integer.toString(data.getBytes().length)); +// Send request + wr = new DataOutputStream(conn.getOutputStream()); + wr.writeBytes(data); + wr.flush(); + wr.close(); + } + + LOG.debug("Status code: " + conn.getResponseCode() + " " + conn.getResponseMessage()); + StringBuilder result = new StringBuilder(); + int responseCode = conn.getResponseCode(); - // Provide filter that will rebuild exception that is sent from server - resource.addFilter(serverExceptionFilter); + if (responseCode == HttpURLConnection.HTTP_OK) { + reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String line = reader.readLine(); + while (line != null) { + result.append(line); + line = reader.readLine(); + } + reader.close(); + } else if (responseCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { + /** + * Client filter to intercepting exceptions sent by sqoop server and + * recreating them on client side. Current implementation will create new + * instance of SqoopException and will attach original error code and message. + * + * Special handling for 500 internal server error in case that server + * has sent us it's exception correctly. We're using default route + * for all other 500 occurrences. + */ + if (conn.getHeaderFields().keySet().contains( + SqoopProtocolConstants.HEADER_SQOOP_INTERNAL_ERROR_CODE)) { - return resource - // Provide name of user executing request. - .header(SqoopProtocolConstants.HEADER_SQOOP_USERNAME, System.getProperty("user.name")) - // Sqoop is using JSON for data transfers - .accept(MediaType.APPLICATION_JSON_TYPE) - // Transfer client locale to return client specific data - .acceptLanguage(Locale.getDefault()); + ThrowableBean ex = new ThrowableBean(); + + result = new StringBuilder(); + reader = new BufferedReader(new InputStreamReader(conn.getErrorStream())); + String line = reader.readLine(); + while (line != null) { + result.append(line); + line = reader.readLine(); + } + reader.close(); + + JSONObject json = (JSONObject) JSONValue.parse(result.toString()); + ex.restore(json); + + throw new SqoopException(ClientError.CLIENT_0001, ex.getThrowable()); + } + } + return result.toString(); + } catch (IOException ex) { + LOG.trace("ERROR: ", ex); + return ""; + } catch (AuthenticationException ex) { + LOG.trace("ERROR: ", ex); + return ""; + } finally { + try { + if (wr != null) { + wr.close(); + } + } catch (IOException e) { + LOG.trace("Cannot close DataOutputStream.", e); + } + try { + if (reader != null) { + reader.close(); + } + } catch (IOException e) { + LOG.trace("Cannot close BufferReader.", e); + } + } } public String get(String url) { - return getBuilder(url).get(String.class); + return doHttpRequest(url, HttpMethod.GET); } public String post(String url, String data) { - return getBuilder(url).post(String.class, data); + return doHttpRequest(url, HttpMethod.POST, data); } public String put(String url, String data) { - return getBuilder(url).put(String.class, data); + return doHttpRequest(url, HttpMethod.PUT, data); } public String delete(String url) { - return getBuilder(url).delete(String.class); - } - - /** - * Client filter to intercepting exceptions sent by sqoop server and - * recreating them on client side. Current implementation will create new - * instance of SqoopException and will attach original error code and message. - */ - private static class ServerExceptionFilter extends ClientFilter { - @Override - public ClientResponse handle(ClientRequest cr) { - ClientResponse resp = getNext().handle(cr); - - // Special handling for 500 internal server error in case that server - // has sent us it's exception correctly. We're using default route - // for all other 500 occurrences. - if(resp.getClientResponseStatus() - == ClientResponse.Status.INTERNAL_SERVER_ERROR) { - - if(resp.getHeaders().containsKey( - SqoopProtocolConstants.HEADER_SQOOP_INTERNAL_ERROR_CODE)) { - - ThrowableBean ex = new ThrowableBean(); - - String responseText = resp.getEntity(String.class); - JSONObject json = (JSONObject) JSONValue.parse(responseText); - ex.restore(json); - - throw new SqoopException(ClientError.CLIENT_0001, ex.getThrowable()); - } - } - - return resp; - } + return doHttpRequest(url, HttpMethod.DELETE); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java index 645555f..26f83a9 100644 --- a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java +++ b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java @@ -68,6 +68,27 @@ public final class AuthenticationConstants { public static final String AUTHENTICATION_KERBEROS_KEYTAB = PREFIX_AUTHENTICATION_KERBEROS_CONFIG + "keytab"; + /** + * All kerberos authentication for http related configuration is prefixed with this: + * <tt>org.apache.sqoop.authentication.kerberos.http.</tt> + */ + public static final String PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG = + PREFIX_AUTHENTICATION_KERBEROS_CONFIG + "http."; + + /** + * The config specifies the kerberos principal for http. + * <tt>org.apache.sqoop.authentication.kerberos.http.principal</tt>. + */ + public static final String AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL = + PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "principal"; + + /** + * The config specifies the kerberos keytab for http. + * <tt>org.apache.sqoop.authentication.kerberos.http.principal</tt>. + */ + public static final String AUTHENTICATION_KERBEROS_HTTP_KEYTAB = + PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "keytab"; + public static enum TYPE {SIMPLE, KERBEROS} private AuthenticationConstants() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java b/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java index 73cd8cf..abb5c90 100644 --- a/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java +++ b/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java @@ -34,7 +34,13 @@ public enum AuthenticationError implements ErrorCode { AUTH_0003("Unable to login using Kerberos keytab and principal"), /** Invalid authentication type {simple, Kerberos}. */ - AUTH_0004("Invalid authentication type"); + AUTH_0004("Invalid authentication type"), + + /** The system was not able to find Kerberos keytab for http in sqoop configuration. */ + AUTH_0005("Unable to find Kerberos keytab for http"), + + /** The system was not able to find Kerberos principal for http in sqoop configuration. */ + AUTH_0006("Unable to find Kerberos principal for http"); private final String message; http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/dist/src/main/server/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties index c5faeca..1666283 100755 --- a/dist/src/main/server/conf/sqoop.properties +++ b/dist/src/main/server/conf/sqoop.properties @@ -149,4 +149,6 @@ org.apache.sqoop.authentication.handler=org.apache.sqoop.security.SimpleAuthenti #org.apache.sqoop.authentication.type=KERBEROS #org.apache.sqoop.authentication.handler=org.apache.sqoop.security.KerberosAuthenticationHandler #org.apache.sqoop.authentication.kerberos.principal=sqoop/_HOST@NOVALOCAL -#org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab \ No newline at end of file +#org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab +#org.apache.sqoop.authentication.kerberos.http.principal=HTTP/_HOST@NOVALOCAL +#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/server/pom.xml ---------------------------------------------------------------------- diff --git a/server/pom.xml b/server/pom.xml index 2a3961c..7b093ac 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -94,6 +94,55 @@ limitations under the License. </dependency> </dependencies> + <!-- Profiles for various supported Hadoop distributions --> + <profiles> + + <!-- Hadoop 1.x --> + <profile> + <id>hadoop100</id> + + <activation> + <property> + <name>hadoop.profile</name> + <value>100</value> + </property> + </activation> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + </profile> + + <!-- Hadoop 2.x (active by default) --> + <profile> + <id>hadoop200</id> + + <activation> + <activeByDefault>true</activeByDefault> + <property> + <name>hadoop.profile</name> + <value>200</value> + </property> + </activation> + + <properties> + <hadoop.profile>200</hadoop.profile> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + </profile> + </profiles> + <build> <finalName>sqoop</finalName> </build> http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java new file mode 100644 index 0000000..a6a79d3 --- /dev/null +++ b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.filter; + +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.sqoop.common.MapContext; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.core.SqoopConfiguration; +import org.apache.sqoop.security.AuthenticationConstants; +import org.apache.sqoop.security.AuthenticationError; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import java.io.IOException; +import java.util.Properties; + +public class SqoopAuthenticationFilter extends AuthenticationFilter { + + @Override + protected Properties getConfiguration(String configPrefix, + FilterConfig filterConfig) throws ServletException { + Properties properties = super.getConfiguration(configPrefix, filterConfig); + MapContext mapContext = SqoopConfiguration.getInstance().getContext(); + String type = mapContext.getString( + AuthenticationConstants.AUTHENTICATION_TYPE).trim(); + + if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.KERBEROS.name())) { + properties.setProperty(AUTH_TYPE, AuthenticationConstants.TYPE.KERBEROS.name().toLowerCase()); + + String keytab = mapContext.getString( + AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB).trim(); + if (keytab.length() == 0) { + throw new SqoopException(AuthenticationError.AUTH_0005, + AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB); + } + + String principal = mapContext.getString( + AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL).trim(); + if (principal.length() == 0) { + throw new SqoopException(AuthenticationError.AUTH_0006, + AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL); + } + + String hostPrincipal = ""; + try { + hostPrincipal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0"); + } catch (IOException e) { + throw new SqoopException(AuthenticationError.AUTH_0006, + AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL); + } + + properties.setProperty(KerberosAuthenticationHandler.PRINCIPAL, hostPrincipal); + properties.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab); + } else if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.SIMPLE.name())) { + properties.setProperty(AUTH_TYPE, PseudoAuthenticationHandler.class.getName()); + } else { + throw new SqoopException(AuthenticationError.AUTH_0004, type); + } + + return properties; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/server/src/main/webapp/WEB-INF/web.xml ---------------------------------------------------------------------- diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml index d405c88..85ae26b 100644 --- a/server/src/main/webapp/WEB-INF/web.xml +++ b/server/src/main/webapp/WEB-INF/web.xml @@ -27,6 +27,17 @@ limitations under the License. <listener-class>org.apache.sqoop.server.ServerInitializer</listener-class> </listener> + <!-- Filter --> + <filter> + <filter-name>authFilter</filter-name> + <filter-class>org.apache.sqoop.filter.SqoopAuthenticationFilter</filter-class> + </filter> + + <filter-mapping> + <filter-name>authFilter</filter-name> + <url-pattern>/*</url-pattern> + </filter-mapping> + <!-- Version servlet --> <servlet> <servlet-name>VersionServlet</servlet-name> http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/shell/pom.xml ---------------------------------------------------------------------- diff --git a/shell/pom.xml b/shell/pom.xml index ba268ce..0249e6e 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -56,11 +56,6 @@ limitations under the License. <version>2.6</version> </dependency> <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - <version>1.11</version> - </dependency> - <dependency> <groupId>jline</groupId> <artifactId>jline</artifactId> <version>0.9.94</version> @@ -75,6 +70,10 @@ limitations under the License. <artifactId>groovy-all</artifactId> <version>1.8.5</version> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + </dependency> </dependencies> <profiles>
