CALCITE-2285 Support client cert keystore for Avatica Client Closes #57
Signed-off-by: Josh Elser <els...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/calcite-avatica/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite-avatica/commit/97c8a161 Tree: http://git-wip-us.apache.org/repos/asf/calcite-avatica/tree/97c8a161 Diff: http://git-wip-us.apache.org/repos/asf/calcite-avatica/diff/97c8a161 Branch: refs/heads/branch-avatica-1.12 Commit: 97c8a1612220e46fe70fed3a92082e55248f868e Parents: 9a37845 Author: Karan Mehta <k.me...@salesforce.com> Authored: Thu Jun 14 12:24:38 2018 -0400 Committer: Josh Elser <els...@apache.org> Committed: Thu Jun 14 12:39:40 2018 -0400 ---------------------------------------------------------------------- .../avatica/BuiltInConnectionProperty.java | 9 ++ .../calcite/avatica/ConnectionConfig.java | 6 + .../calcite/avatica/ConnectionConfigImpl.java | 17 +++ .../remote/AvaticaCommonsHttpClientImpl.java | 108 ++++++++++++----- .../remote/AvaticaHttpClientFactoryImpl.java | 15 ++- .../avatica/remote/KeyStoreConfigurable.java | 39 ++++++ ...aCommonsHttpClientImplSocketFactoryTest.java | 120 +++++++++++++++++++ core/src/test/resources/log4j.properties | 24 ++++ 8 files changed, 310 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/97c8a161/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java b/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java index 1da7025..a1babb3 100644 --- a/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java +++ b/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java @@ -76,6 +76,15 @@ public enum BuiltInConnectionProperty implements ConnectionProperty { /** Password for the truststore */ TRUSTSTORE_PASSWORD("truststore_password", Type.STRING, null, false), + /** Keystore for MTLS authentication */ + KEYSTORE("keystore", Type.STRING, null, false), + + /** Password for the keystore */ + KEYSTORE_PASSWORD("keystore_password", Type.STRING, null, false), + + /** Password for the key inside keystore */ + KEY_PASSWORD("key_password", Type.STRING, null, false), + HOSTNAME_VERIFICATION("hostname_verification", Type.ENUM, HostnameVerification.STRICT, HostnameVerification.class, false); http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/97c8a161/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java index cd18ec5..bbbfa87 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java @@ -54,6 +54,12 @@ public interface ConnectionConfig { File truststore(); /** @see BuiltInConnectionProperty#TRUSTSTORE_PASSWORD */ String truststorePassword(); + /** @see BuiltInConnectionProperty#KEYSTORE */ + File keystore(); + /** @see BuiltInConnectionProperty#KEYSTORE_PASSWORD */ + String keystorePassword(); + /** @see BuiltInConnectionProperty#KEY_PASSWORD */ + String keyPassword(); /** @see BuiltInConnectionProperty#HOSTNAME_VERIFICATION */ HostnameVerification hostnameVerification(); } http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/97c8a161/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java index 94cdc51..36cdf61 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java @@ -106,6 +106,23 @@ public class ConnectionConfigImpl implements ConnectionConfig { return BuiltInConnectionProperty.TRUSTSTORE_PASSWORD.wrap(properties).getString(); } + public File keystore() { + String filename = BuiltInConnectionProperty.KEYSTORE.wrap(properties).getString(); + if (null == filename) { + return null; + } + return new File(filename); + } + + public String keystorePassword() { + return BuiltInConnectionProperty.KEYSTORE_PASSWORD.wrap(properties).getString(); + } + + public String keyPassword() { + return BuiltInConnectionProperty.KEY_PASSWORD.wrap(properties).getString(); + + } + public HostnameVerification hostnameVerification() { return BuiltInConnectionProperty.HOSTNAME_VERIFICATION.wrap(properties) .getEnum(HostnameVerification.class); http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/97c8a161/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImpl.java b/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImpl.java index 5576847..e7e6aa0 100644 --- a/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImpl.java @@ -28,6 +28,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.Lookup; +import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; @@ -42,6 +43,7 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; @@ -64,7 +66,8 @@ import javax.net.ssl.SSLContext; * sent and received across the wire. */ public class AvaticaCommonsHttpClientImpl implements AvaticaHttpClient, - UsernamePasswordAuthenticateable, TrustStoreConfigurable, HostnameVerificationConfigurable { + UsernamePasswordAuthenticateable, TrustStoreConfigurable, + KeyStoreConfigurable, HostnameVerificationConfigurable { private static final Logger LOG = LoggerFactory.getLogger(AvaticaCommonsHttpClientImpl.class); // Some basic exposed configurations @@ -78,14 +81,19 @@ public class AvaticaCommonsHttpClientImpl implements AvaticaHttpClient, protected final URI uri; protected BasicAuthCache authCache; protected CloseableHttpClient client; - PoolingHttpClientConnectionManager pool; + protected Registry<ConnectionSocketFactory> socketFactoryRegistry; + protected PoolingHttpClientConnectionManager pool; protected UsernamePasswordCredentials credentials = null; protected CredentialsProvider credentialsProvider = null; protected Lookup<AuthSchemeProvider> authRegistry = null; + protected boolean configureHttpsSocket = false; protected File truststore = null; + protected File keystore = null; protected String truststorePassword = null; + protected String keystorePassword = null; + protected String keyPassword = null; protected HostnameVerification hostnameVerification = null; public AvaticaCommonsHttpClientImpl(URL url) { @@ -95,29 +103,15 @@ public class AvaticaCommonsHttpClientImpl implements AvaticaHttpClient, } private void initializeClient() { - SSLConnectionSocketFactory sslFactory = null; - if (null != truststore && null != truststorePassword) { - try { - SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial( - truststore, truststorePassword.toCharArray()).build(); - - final HostnameVerifier verifier = getHostnameVerifier(hostnameVerification); - - sslFactory = new SSLConnectionSocketFactory(sslcontext, verifier); - } catch (Exception e) { - throw new RuntimeException(e); - } - } else { - LOG.debug("Not configuring HTTPS because of missing truststore/password"); - } + socketFactoryRegistry = this.configureSocketFactories(); + configureConnectionPool(socketFactoryRegistry); + this.authCache = new BasicAuthCache(); + // A single thread-safe HttpClient, pooling connections via the ConnectionManager + this.client = HttpClients.custom().setConnectionManager(pool).build(); + } - RegistryBuilder<ConnectionSocketFactory> registryBuilder = RegistryBuilder.create(); - registryBuilder.register("http", PlainConnectionSocketFactory.getSocketFactory()); - // Only register the SSL factory when provided - if (null != sslFactory) { - registryBuilder.register("https", sslFactory); - } - pool = new PoolingHttpClientConnectionManager(registryBuilder.build()); + protected void configureConnectionPool(Registry<ConnectionSocketFactory> registry) { + pool = new PoolingHttpClientConnectionManager(registry); // Increase max total connection to 100 final String maxCnxns = System.getProperty(MAX_POOLED_CONNECTIONS_KEY, @@ -127,11 +121,57 @@ public class AvaticaCommonsHttpClientImpl implements AvaticaHttpClient, final String maxCnxnsPerRoute = System.getProperty(MAX_POOLED_CONNECTION_PER_ROUTE_KEY, MAX_POOLED_CONNECTION_PER_ROUTE_DEFAULT); pool.setDefaultMaxPerRoute(Integer.parseInt(maxCnxnsPerRoute)); + } - this.authCache = new BasicAuthCache(); + protected Registry<ConnectionSocketFactory> configureSocketFactories() { + RegistryBuilder<ConnectionSocketFactory> registryBuilder = RegistryBuilder.create(); + if (host.getSchemeName().equalsIgnoreCase("https")) { + configureHttpsRegistry(registryBuilder); + } else { + configureHttpRegistry(registryBuilder); + } + return registryBuilder.build(); + } - // A single thread-safe HttpClient, pooling connections via the ConnectionManager - this.client = HttpClients.custom().setConnectionManager(pool).build(); + protected void configureHttpsRegistry(RegistryBuilder<ConnectionSocketFactory> registryBuilder) { + if (!configureHttpsSocket) { + LOG.debug("HTTPS Socket not being configured because no truststore/keystore provided"); + return; + } + + try { + SSLContext sslContext = getSSLContext(); + final HostnameVerifier verifier = getHostnameVerifier(hostnameVerification); + SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext, verifier); + registryBuilder.register("https", sslFactory); + } catch (Exception e) { + LOG.error("HTTPS registry configuration failed"); + throw new RuntimeException(e); + } + } + + private SSLContext getSSLContext() throws Exception { + SSLContextBuilder sslContextBuilder = SSLContexts.custom(); + if (null != truststore && null != truststorePassword) { + loadTrustStore(sslContextBuilder); + } + if (null != keystore && null != keystorePassword && null != keyPassword) { + loadKeyStore(sslContextBuilder); + } + return sslContextBuilder.build(); + } + + protected void loadKeyStore(SSLContextBuilder sslContextBuilder) throws Exception { + sslContextBuilder.loadKeyMaterial(keystore, + keystorePassword.toCharArray(), keyPassword.toCharArray()); + } + + protected void loadTrustStore(SSLContextBuilder sslContextBuilder) throws Exception { + sslContextBuilder.loadTrustMaterial(truststore, truststorePassword.toCharArray()); + } + + protected void configureHttpRegistry(RegistryBuilder<ConnectionSocketFactory> registryBuilder) { + registryBuilder.register("http", PlainConnectionSocketFactory.getSocketFactory()); } /** @@ -244,11 +284,25 @@ public class AvaticaCommonsHttpClientImpl implements AvaticaHttpClient, "Truststore is must be an existing, regular file: " + truststore); } this.truststorePassword = Objects.requireNonNull(password); + configureHttpsSocket = true; + initializeClient(); + } + + @Override public void setKeyStore(File keystore, String keystorepassword, String keypassword) { + this.keystore = Objects.requireNonNull(keystore); + if (!keystore.exists() || !keystore.isFile()) { + throw new IllegalArgumentException( + "Keystore is must be an existing, regular file: " + keystore); + } + this.keystorePassword = Objects.requireNonNull(keystorepassword); + this.keyPassword = Objects.requireNonNull(keypassword); + configureHttpsSocket = true; initializeClient(); } @Override public void setHostnameVerification(HostnameVerification verification) { this.hostnameVerification = Objects.requireNonNull(verification); + configureHttpsSocket = true; initializeClient(); } } http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/97c8a161/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaHttpClientFactoryImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaHttpClientFactoryImpl.java b/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaHttpClientFactoryImpl.java index f352ac1..8f305dc 100644 --- a/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaHttpClientFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/remote/AvaticaHttpClientFactoryImpl.java @@ -74,12 +74,25 @@ public class AvaticaHttpClientFactoryImpl implements AvaticaHttpClientFactory { File truststore = config.truststore(); String truststorePassword = config.truststorePassword(); if (null != truststore && null != truststorePassword) { - ((TrustStoreConfigurable) client).setTrustStore(truststore, truststorePassword); + ((TrustStoreConfigurable) client) + .setTrustStore(truststore, truststorePassword); } } else { LOG.debug("{} is not capable of SSL/TLS communication", client.getClass().getName()); } + if (client instanceof KeyStoreConfigurable) { + File keystore = config.keystore(); + String keystorePassword = config.keystorePassword(); + String keyPassword = config.keyPassword(); + if (null != keystore && null != keystorePassword && null != keyPassword) { + ((KeyStoreConfigurable) client) + .setKeyStore(keystore, keystorePassword, keyPassword); + } + } else { + LOG.debug("{} is not capable of Mutual authentication", client.getClass().getName()); + } + // Set the SSL hostname verification if the client supports it if (client instanceof HostnameVerificationConfigurable) { ((HostnameVerificationConfigurable) client) http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/97c8a161/core/src/main/java/org/apache/calcite/avatica/remote/KeyStoreConfigurable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/KeyStoreConfigurable.java b/core/src/main/java/org/apache/calcite/avatica/remote/KeyStoreConfigurable.java new file mode 100644 index 0000000..eaffd2a --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/KeyStoreConfigurable.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote; + +import java.io.File; + +/** + * Allows a keystore (and keystorepassword, keypassword) to be + * provided to enable MTLS authentication + */ +public interface KeyStoreConfigurable { + + /** + * Sets a keystore containing the collection of client side certificates + * to use for HTTPS mutual authentication along with + * password for keystore and password for key + * + * @param keystore The keystore on the local filesystem + * @param keystorepassword The keystore's password + * @param keypassword The key's password + */ + void setKeyStore(File keystore, String keystorepassword, String keypassword); +} + +// End KeyStoreConfigurable.java http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/97c8a161/core/src/test/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImplSocketFactoryTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImplSocketFactoryTest.java b/core/src/test/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImplSocketFactoryTest.java new file mode 100644 index 0000000..a75222a --- /dev/null +++ b/core/src/test/java/org/apache/calcite/avatica/remote/AvaticaCommonsHttpClientImplSocketFactoryTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote; + +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.ssl.SSLContextBuilder; + +import org.junit.Test; + +import java.io.File; +import java.net.URL; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Tests to verify loading of truststore/keystore in AvaticaCommonsHttpClientImpl + */ +public class AvaticaCommonsHttpClientImplSocketFactoryTest { + + private static final String HTTP_REGISTRY = "http"; + private static final String HTTPS_REGISTRY = "https"; + + private URL url; + private AvaticaCommonsHttpClientImpl client; + private File storeFile; + private String password; + + @Test public void testPlainSocketFactory() throws Exception { + configureHttpClient(); + assertFalse("Https socket should not be configured" + + " without truststore/keystore", client.configureHttpsSocket); + verifyFactoryInstance(client, HTTP_REGISTRY, PlainConnectionSocketFactory.class); + verifyFactoryInstance(client, HTTPS_REGISTRY, null); + verify(client, times(0)).loadTrustStore(any(SSLContextBuilder.class)); + verify(client, times(0)).loadKeyStore(any(SSLContextBuilder.class)); + } + + @Test public void testTrustStoreLoadedInFactory() throws Exception { + configureHttpsClient(); + client.setTrustStore(storeFile, password); + assertTrue("Https socket should be configured" + + " with truststore", client.configureHttpsSocket); + verifyFactoryInstance(client, HTTP_REGISTRY, null); + verifyFactoryInstance(client, HTTPS_REGISTRY, SSLConnectionSocketFactory.class); + verify(client, times(1)).configureSocketFactories(); + verify(client, times(1)).loadTrustStore(any(SSLContextBuilder.class)); + verify(client, times(0)).loadKeyStore(any(SSLContextBuilder.class)); + } + + @Test public void testKeyStoreLoadedInFactory() throws Exception { + configureHttpsClient(); + client.setKeyStore(storeFile, password, password); + assertTrue("Https socket should be configured" + + " with keystore", client.configureHttpsSocket); + verifyFactoryInstance(client, HTTP_REGISTRY, null); + verifyFactoryInstance(client, HTTPS_REGISTRY, SSLConnectionSocketFactory.class); + verify(client, times(1)).configureSocketFactories(); + verify(client, times(0)).loadTrustStore(any(SSLContextBuilder.class)); + verify(client, times(1)).loadKeyStore(any(SSLContextBuilder.class)); + } + + private void configureHttpClient() throws Exception { + url = new URL("http://fake_url.com";); + configureClient(); + } + + private void configureHttpsClient() throws Exception { + url = new URL("https://fake_url.com";); + configureClient(); + } + + private void configureClient() throws Exception { + client = spy(new AvaticaCommonsHttpClientImpl(url)); + // storeFile can be used as either Keystore/Truststore + storeFile = mock(File.class); + when(storeFile.exists()).thenReturn(true); + when(storeFile.isFile()).thenReturn(true); + password = ""; + + doNothing().when(client).loadTrustStore(any(SSLContextBuilder.class)); + doNothing().when(client).loadKeyStore(any(SSLContextBuilder.class)); + } + + <T> void verifyFactoryInstance(AvaticaCommonsHttpClientImpl client, + String registry, Class<T> expected) { + ConnectionSocketFactory factory = client.socketFactoryRegistry.lookup(registry); + if (expected == null) { + assertTrue("Factory for registry " + registry + " expected as null", factory == null); + } else { + assertTrue("Factory for registry " + registry + " expected of type " + expected.getName(), + expected.equals(factory.getClass())); + } + } +} + +// End AvaticaCommonsHttpClientImplSocketFactoryTest.java http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/97c8a161/core/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties new file mode 100644 index 0000000..834e2db --- /dev/null +++ b/core/src/test/resources/log4j.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root logger is configured at INFO and is sent to A1 +log4j.rootLogger=INFO, A1 + +# A1 goes to the console +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# Set the pattern for each log message +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p - %m%n
