SLIDER-878 use own connection factory with controlled order on connection operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1cecc3fc Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1cecc3fc Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1cecc3fc Branch: refs/heads/develop Commit: 1cecc3fcb0e80de48e0ea7ba54e607421ffea3fe Parents: cac169a Author: Steve Loughran <[email protected]> Authored: Mon May 18 20:36:16 2015 +0100 Committer: Steve Loughran <[email protected]> Committed: Mon May 18 20:36:16 2015 +0100 ---------------------------------------------------------------------- pom.xml | 7 +- .../restclient/SliderURLConnectionFactory.java | 176 +++++++++++++++++++ .../core/restclient/UgiJerseyBinding.java | 3 +- .../restclient/UrlConnectionOperations.java | 25 +-- 4 files changed, 186 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6a9152c..53b948d 100644 --- a/pom.xml +++ b/pom.xml @@ -1591,7 +1591,7 @@ <!-- hadoop branch-2.7 builds --> <id>branch-2.7</id> <properties> - <hadoop.version>2.7.0-SNAPSHOT</hadoop.version> + <hadoop.version>2.7.1-SNAPSHOT</hadoop.version> </properties> </profile> @@ -1615,7 +1615,10 @@ <!-- Java 8 build --> <id>java8</id> <properties> - <project.java.src.version>1.8</project.java.src.version> + <project.java.src.version>1.8</project.java.src.version> +<!-- + <jersey.version>1.19</jersey.version> +--> </properties> </profile> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java b/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java new file mode 100644 index 0000000..e453f52 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.restclient; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; +import org.apache.hadoop.security.ssl.SSLFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLSocketFactory; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.security.GeneralSecurityException; + +/** + * Factory for URL connections; used behind the scenes in the Jersey integration. + * <p> + * Derived from the WebHDFS implementation. + */ +public class SliderURLConnectionFactory { + private static final Logger log = + LoggerFactory.getLogger(SliderURLConnectionFactory.class); + + /** + * Timeout for socket connects and reads + */ + public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute + private final ConnectionConfigurator connConfigurator; + + private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator(); + + /** + * Construct a new URLConnectionFactory based on the configuration. It will + * try to load SSL certificates when it is specified. + */ + public static SliderURLConnectionFactory newInstance(Configuration conf) { + ConnectionConfigurator conn; + try { + conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf); + } catch (Exception e) { + log.debug("Cannot load customized SSL configuration.", e); + conn = DEFAULT_CONFIGURATOR; + } + return new SliderURLConnectionFactory(conn); + } + + private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) { + this.connConfigurator = connConfigurator; + } + + /** + * Create a new ConnectionConfigurator for SSL connections + */ + private static ConnectionConfigurator newSslConnConfigurator(final int timeout, + Configuration conf) throws IOException, GeneralSecurityException { + final SSLFactory factory; + final SSLSocketFactory sf; + final HostnameVerifier hv; + + factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + factory.init(); + sf = factory.createSSLSocketFactory(); + hv = factory.getHostnameVerifier(); + + return new ConnectionConfigurator() { + @Override + public HttpURLConnection configure(HttpURLConnection conn) + throws IOException { + if (conn instanceof HttpsURLConnection) { + HttpsURLConnection c = (HttpsURLConnection) conn; + c.setSSLSocketFactory(sf); + c.setHostnameVerifier(hv); + } + SliderURLConnectionFactory.setupConnection(conn, timeout); + return conn; + } + }; + } + + /** + * Opens a url with read and connect timeouts + * + * @param url + * to open + * @return URLConnection + * @throws IOException + */ + public URLConnection openConnection(URL url) throws IOException { + try { + return openConnection(url, false); + } catch (AuthenticationException e) { + // Unreachable + return null; + } + } + + /** + * Opens a url with read and connect timeouts + * + * @param url + * URL to open + * @param isSpnego + * whether the url should be authenticated via SPNEGO + * @return URLConnection + * @throws IOException + * @throws AuthenticationException + */ + public URLConnection openConnection(URL url, boolean isSpnego) + throws IOException, AuthenticationException { + if (isSpnego) { + log.debug("open AuthenticatedURL connection {}", url); + UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab(); + final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); + return new AuthenticatedURL(new KerberosUgiAuthenticator(), + connConfigurator).openConnection(url, authToken); + } else { + log.debug("open URL connection {}", url); + URLConnection connection = url.openConnection(); + if (connection instanceof HttpURLConnection) { + connConfigurator.configure((HttpURLConnection) connection); + } + return connection; + } + } + + /** + * Sets connection parameters on the given URLConnection + * + * @param connection + * URLConnection to set + * @param socketTimeout + * the connection and read timeout of the connection. + */ + private static void setupConnection(URLConnection connection, int socketTimeout) { + connection.setConnectTimeout(socketTimeout); + connection.setReadTimeout(socketTimeout); + connection.setUseCaches(false); + if (connection instanceof HttpURLConnection) { + ((HttpURLConnection) connection).setInstanceFollowRedirects(true); + } + } + + private static class BasicConfigurator implements ConnectionConfigurator { + @Override + public HttpURLConnection configure(HttpURLConnection conn) + throws IOException { + SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT); + return conn; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java index 2526684..d19c621 100644 --- a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java +++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java @@ -88,8 +88,7 @@ public class UgiJerseyBinding implements try { // open a connection handling status codes and so redirections // but as it opens a connection, it's less useful than you think. -// return operations.openConnectionRedirecting(url); - + return operations.openConnection(url); } catch (AuthenticationException e) { throw new IOException(e); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java index 49fe5ea..e0d2315 100644 --- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java @@ -22,7 +22,6 @@ import com.google.common.base.Preconditions; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; @@ -31,24 +30,21 @@ import org.apache.hadoop.yarn.webapp.NotFoundException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.core.HttpHeaders; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; -import java.net.URI; import java.net.URL; /** - * Operations on the JDK UrlConnection class. This uses WebHDFS - * methods to set up the operations. + * Operations on the JDK UrlConnection class. + * */ public class UrlConnectionOperations extends Configured { private static final Logger log = LoggerFactory.getLogger(UrlConnectionOperations.class); - private URLConnectionFactory connectionFactory; + private SliderURLConnectionFactory connectionFactory; private boolean useSpnego = false; @@ -59,8 +55,7 @@ public class UrlConnectionOperations extends Configured { */ public UrlConnectionOperations(Configuration conf) { super(conf); - connectionFactory = URLConnectionFactory - .newDefaultURLConnectionFactory(conf); + connectionFactory = SliderURLConnectionFactory.newInstance(conf); if (UserGroupInformation.isSecurityEnabled()) { log.debug("SPNEGO is enabled"); setUseSpnego(true); @@ -88,18 +83,6 @@ public class UrlConnectionOperations extends Configured { public HttpURLConnection openConnection(URL url) throws IOException, AuthenticationException { - - HttpURLConnection conn = innerOpenConnection(url); - conn.setUseCaches(false); - conn.setInstanceFollowRedirects(true); - return conn; - } - - - - protected HttpURLConnection innerOpenConnection(URL url) throws - IOException, - AuthenticationException { Preconditions.checkArgument(url.getPort() != 0, "no port"); return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego); }
