This is an automated email from the ASF dual-hosted git repository. apucher pushed a commit to branch pinot-internode-tls in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit b24bdeda8a5e702a0b82022ff42576b94bf31816 Author: Alexander Pucher <[email protected]> AuthorDate: Wed Jan 6 19:17:24 2021 -0800 unified client and internode TLS --- .../broker/broker/BrokerAdminApiApplication.java | 28 ++--- .../broker/broker/helix/HelixBrokerStarter.java | 6 +- .../SingleConnectionBrokerRequestHandler.java | 6 +- .../LiteralOnlyBrokerRequestTest.java | 4 +- .../spark/connector/PinotServerDataFetcher.scala | 1 + .../apache/pinot/controller/ControllerStarter.java | 7 ++ .../api/ControllerAdminApiApplication.java | 34 +++--- .../controller/api/listeners/ListenerConfig.java | 39 +++--- .../controller/api/listeners/TlsConfiguration.java | 63 ---------- .../pinot/controller/util/ListenerConfigUtil.java | 29 ++--- .../controller/util/ListenerConfigUtilTest.java | 19 +-- .../apache/pinot/core/transport/QueryRouter.java | 19 +++ .../apache/pinot/core/transport/QueryServer.java | 53 +++++++++ .../pinot/core/transport/ServerChannels.java | 45 +++++++ .../org/apache/pinot/core/transport/TlsConfig.java | 61 ++++++++++ .../java/org/apache/pinot/core/util/TlsUtils.java | 132 +++++++++++++++++++++ .../org/apache/pinot/server/conf/ServerConf.java | 4 + .../pinot/server/starter/ServerInstance.java | 11 +- 18 files changed, 408 insertions(+), 153 deletions(-) diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java index 821e4aa..5d4b13a 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java @@ -29,6 +29,8 @@ import org.apache.pinot.broker.requesthandler.BrokerRequestHandler; import org.apache.pinot.broker.routing.RoutingManager; import org.apache.pinot.common.metrics.BrokerMetrics; import org.apache.pinot.common.utils.CommonConstants; +import org.apache.pinot.core.transport.TlsConfig; +import org.apache.pinot.core.util.TlsUtils; import org.apache.pinot.spi.env.PinotConfiguration; import org.glassfish.grizzly.http.server.CLStaticHttpHandler; import org.glassfish.grizzly.http.server.HttpHandler; @@ -73,7 +75,6 @@ public class BrokerAdminApiApplication extends ResourceConfig { Preconditions.checkArgument(brokerQueryPort > 0, "broker client port must be > 0"); _baseUri = URI.create(String.format("%s://0.0.0.0:%d/", getBrokerClientProtocol(brokerConf), brokerQueryPort)); - _httpServer = buildHttpsServer(brokerConf); setupSwagger(); } @@ -81,31 +82,26 @@ public class BrokerAdminApiApplication extends ResourceConfig { private HttpServer buildHttpsServer(PinotConfiguration brokerConf) { boolean isSecure = CommonConstants.HTTPS_PROTOCOL.equals(getBrokerClientProtocol(brokerConf)); + TlsConfig tlsConfig = TlsUtils.extractTlsConfig(brokerConf, "pinot.broker.client"); + tlsConfig.setEnabled(isSecure); + if (isSecure) { - return GrizzlyHttpServerFactory.createHttpServer(_baseUri, this, true, buildSSLConfig(brokerConf)); + return GrizzlyHttpServerFactory.createHttpServer(_baseUri, this, true, buildSSLConfig(tlsConfig)); } return GrizzlyHttpServerFactory.createHttpServer(_baseUri, this); } - private SSLEngineConfigurator buildSSLConfig(PinotConfiguration brokerConf) { + private SSLEngineConfigurator buildSSLConfig(TlsConfig tlsConfig) { SSLContextConfigurator sslContextConfigurator = new SSLContextConfigurator(); - sslContextConfigurator.setKeyStoreFile(brokerConf.getProperty( - CommonConstants.Broker.CONFIG_OF_BROKER_CLIENT_TLS_KEYSTORE_PATH)); - sslContextConfigurator.setKeyStorePass(brokerConf.getProperty( - CommonConstants.Broker.CONFIG_OF_BROKER_CLIENT_TLS_KEYSTORE_PASSWORD)); - sslContextConfigurator.setTrustStoreFile(brokerConf.getProperty( - CommonConstants.Broker.CONFIG_OF_BROKER_CLIENT_TLS_TRUSTSTORE_PATH)); - sslContextConfigurator.setTrustStorePass(brokerConf.getProperty( - CommonConstants.Broker.CONFIG_OF_BROKER_CLIENT_TLS_TRUSTSTORE_PASSWORD)); - - boolean requiresClientAuth = brokerConf.getProperty( - CommonConstants.Broker.CONFIG_OF_BROKER_CLIENT_TLS_CLIENT_AUTH, - CommonConstants.Broker.DEFAULT_BROKER_CLIENT_TLS_CLIENT_AUTH); + sslContextConfigurator.setKeyStoreFile(tlsConfig.getKeyStorePath()); + sslContextConfigurator.setKeyStorePass(tlsConfig.getKeyStorePassword()); + sslContextConfigurator.setTrustStoreFile(tlsConfig.getTrustStorePath()); + sslContextConfigurator.setTrustStorePass(tlsConfig.getTrustStorePassword()); return new SSLEngineConfigurator(sslContextConfigurator).setClientMode(false) - .setWantClientAuth(requiresClientAuth).setEnabledProtocols(new String[] { "TLSv1.2" }); + .setNeedClientAuth(tlsConfig.isClientAuth()).setEnabledProtocols(new String[] { "TLSv1.2" }); } private static String getBrokerClientProtocol(PinotConfiguration brokerConf) { diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/HelixBrokerStarter.java b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/HelixBrokerStarter.java index 746bf7a..c31c3b7 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/HelixBrokerStarter.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/HelixBrokerStarter.java @@ -60,6 +60,8 @@ import org.apache.pinot.common.utils.NetUtil; import org.apache.pinot.common.utils.ServiceStatus; import org.apache.pinot.common.utils.config.TagNameUtils; import org.apache.pinot.common.utils.helix.TableCache; +import org.apache.pinot.core.transport.TlsConfig; +import org.apache.pinot.core.util.TlsUtils; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.services.ServiceRole; import org.apache.pinot.spi.services.ServiceStartable; @@ -238,9 +240,11 @@ public class HelixBrokerStarter implements ServiceStartable { // Initialize FunctionRegistry before starting the broker request handler FunctionRegistry.init(); TableCache tableCache = new TableCache(_propertyStore, caseInsensitive); + // Configure TLS + TlsConfig tlsConfig = TlsUtils.extractTlsConfig(_brokerConf, "pinot.broker.netty"); _brokerRequestHandler = new SingleConnectionBrokerRequestHandler(_brokerConf, _routingManager, _accessControlFactory, queryQuotaManager, - tableCache, _brokerMetrics); + tableCache, _brokerMetrics, tlsConfig); int brokerQueryPort = _brokerConf.getProperty(Helix.KEY_OF_BROKER_QUERY_PORT, Helix.DEFAULT_BROKER_QUERY_PORT); LOGGER.info("Starting broker admin application on port: {}", brokerQueryPort); diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java index cbffeb1..ec24e66 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java @@ -44,6 +44,8 @@ import org.apache.pinot.core.transport.QueryRouter; import org.apache.pinot.core.transport.ServerInstance; import org.apache.pinot.core.transport.ServerResponse; import org.apache.pinot.core.transport.ServerRoutingInstance; +import org.apache.pinot.core.transport.TlsConfig; +import org.apache.pinot.core.util.TlsUtils; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.utils.builder.TableNameBuilder; @@ -58,9 +60,9 @@ public class SingleConnectionBrokerRequestHandler extends BaseBrokerRequestHandl public SingleConnectionBrokerRequestHandler(PinotConfiguration config, RoutingManager routingManager, AccessControlFactory accessControlFactory, QueryQuotaManager queryQuotaManager, TableCache tableCache, - BrokerMetrics brokerMetrics) { + BrokerMetrics brokerMetrics, TlsConfig tlsConfig) { super(config, routingManager, accessControlFactory, queryQuotaManager, tableCache, brokerMetrics); - _queryRouter = new QueryRouter(_brokerId, brokerMetrics); + _queryRouter = new QueryRouter(_brokerId, brokerMetrics, tlsConfig); } @Override diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java index e23435e..65896d2 100644 --- a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java +++ b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java @@ -92,7 +92,7 @@ public class LiteralOnlyBrokerRequestTest { throws Exception { SingleConnectionBrokerRequestHandler requestHandler = new SingleConnectionBrokerRequestHandler(new PinotConfiguration(), null, null, null, null, - new BrokerMetrics("", new MetricsRegistry(), true, Collections.emptySet())); + new BrokerMetrics("", new MetricsRegistry(), true, Collections.emptySet()), null); long randNum = RANDOM.nextLong(); byte[] randBytes = new byte[12]; RANDOM.nextBytes(randBytes); @@ -119,7 +119,7 @@ public class LiteralOnlyBrokerRequestTest { throws Exception { SingleConnectionBrokerRequestHandler requestHandler = new SingleConnectionBrokerRequestHandler(new PinotConfiguration(), null, null, null, null, - new BrokerMetrics("", new MetricsRegistry(), true, Collections.emptySet())); + new BrokerMetrics("", new MetricsRegistry(), true, Collections.emptySet()), null); long currentTsMin = System.currentTimeMillis(); JsonNode request = new ObjectMapper().readTree( "{\"sql\":\"SELECT now() as currentTs, fromDateTime('2020-01-01 UTC', 'yyyy-MM-dd z') as firstDayOf2020\"}"); diff --git a/pinot-connectors/pinot-spark-connector/src/main/scala/org/apache/pinot/connector/spark/connector/PinotServerDataFetcher.scala b/pinot-connectors/pinot-spark-connector/src/main/scala/org/apache/pinot/connector/spark/connector/PinotServerDataFetcher.scala index 6b938cb..1c103b4 100644 --- a/pinot-connectors/pinot-spark-connector/src/main/scala/org/apache/pinot/connector/spark/connector/PinotServerDataFetcher.scala +++ b/pinot-connectors/pinot-spark-connector/src/main/scala/org/apache/pinot/connector/spark/connector/PinotServerDataFetcher.scala @@ -48,6 +48,7 @@ private[pinot] class PinotServerDataFetcher( private val metricsRegistry = new MetricsRegistry() private val brokerMetrics = new BrokerMetrics(metricsRegistry) private val queryRouter = new QueryRouter(brokerId, brokerMetrics) + // TODO add support for TLS-secured server def fetchData(): List[DataTable] = { val routingTableForRequest = createRoutingTableForRequest() diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerStarter.java b/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerStarter.java index fb46da9..c1e1efc 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerStarter.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerStarter.java @@ -83,6 +83,7 @@ import org.apache.pinot.controller.validation.OfflineSegmentIntervalChecker; import org.apache.pinot.controller.validation.RealtimeSegmentValidationManager; import org.apache.pinot.core.periodictask.PeriodicTask; import org.apache.pinot.core.periodictask.PeriodicTaskScheduler; +import org.apache.pinot.core.util.TlsUtils; import org.apache.pinot.spi.crypt.PinotCrypterFactory; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.filesystem.PinotFSFactory; @@ -413,6 +414,12 @@ public class ControllerStarter implements ServiceStartable { } }); + // install default SSL context if necessary + if (CommonConstants.HTTPS_PROTOCOL.equals(_config.getProperty(ControllerConf.CONTROLLER_BROKER_PROTOCOL))) { + LOGGER.info("Installing default SSL context for broker relay requests"); + TlsUtils.installDefaultSSLSocketFactory(TlsUtils.extractTlsConfig(_config, "controller.broker")); + } + _adminApp.start(_listenerConfigs); _listenerConfigs.stream().forEach(listenerConfig -> LOGGER.info("Controller services available at {}://{}:{}/", diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java index 5c2c8e3..a18553e 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java @@ -18,23 +18,20 @@ */ package org.apache.pinot.controller.api; +import com.google.common.base.Preconditions; +import io.swagger.jaxrs.config.BeanConfig; import java.io.IOException; -import java.net.InetAddress; import java.net.URI; import java.net.URL; import java.net.URLClassLoader; -import java.net.UnknownHostException; -import java.util.Collection; import java.util.List; import java.util.stream.Collectors; - import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.container.ContainerResponseContext; import javax.ws.rs.container.ContainerResponseFilter; - -import org.apache.pinot.controller.api.listeners.ListenerConfig; -import org.apache.pinot.controller.api.listeners.TlsConfiguration; import org.apache.pinot.common.utils.CommonConstants; +import org.apache.pinot.controller.api.listeners.ListenerConfig; +import org.apache.pinot.core.transport.TlsConfig; import org.glassfish.grizzly.http.server.CLStaticHttpHandler; import org.glassfish.grizzly.http.server.HttpServer; import org.glassfish.grizzly.http.server.NetworkListener; @@ -50,10 +47,6 @@ import org.glassfish.jersey.server.ResourceConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; - -import io.swagger.jaxrs.config.BeanConfig; - public class ControllerAdminApiApplication extends ResourceConfig { private static final Logger LOGGER = LoggerFactory.getLogger(ControllerAdminApiApplication.class); @@ -76,16 +69,16 @@ public class ControllerAdminApiApplication extends ResourceConfig { // property("jersey.config.server.tracing.threshold", "VERBOSE"); } - private SSLEngineConfigurator buildSSLEngineConfigurator(TlsConfiguration tlsConfiguration) { + private SSLEngineConfigurator buildSSLEngineConfigurator(TlsConfig tlsConfig) { SSLContextConfigurator sslContextConfigurator = new SSLContextConfigurator(); - sslContextConfigurator.setKeyStoreFile(tlsConfiguration.getKeyStorePath()); - sslContextConfigurator.setKeyStorePass(tlsConfiguration.getKeyStorePassword()); - sslContextConfigurator.setTrustStoreFile(tlsConfiguration.getTrustStorePath()); - sslContextConfigurator.setTrustStorePass(tlsConfiguration.getTrustStorePassword()); + sslContextConfigurator.setKeyStoreFile(tlsConfig.getKeyStorePath()); + sslContextConfigurator.setKeyStorePass(tlsConfig.getKeyStorePassword()); + sslContextConfigurator.setTrustStoreFile(tlsConfig.getTrustStorePath()); + sslContextConfigurator.setTrustStorePass(tlsConfig.getTrustStorePassword()); return new SSLEngineConfigurator(sslContextConfigurator).setClientMode(false) - .setWantClientAuth(tlsConfiguration.isRequiresClientAuth()).setEnabledProtocols(new String[] { "TLSv1.2 " }); + .setNeedClientAuth(tlsConfig.isClientAuth()).setEnabledProtocols(new String[] { "TLSv1.2" }); } public void registerBinder(AbstractBinder binder) { @@ -100,10 +93,11 @@ public class ControllerAdminApiApplication extends ResourceConfig { .setThreadFactory(new ThreadFactoryBuilder().setNameFormat("grizzly-http-server-%d") .setUncaughtExceptionHandler(new JerseyProcessingUncaughtExceptionHandler()).build()); - listener.setSecure(listenerConfig.getTlsConfiguration() != null); - if (listener.isSecure()) { - listener.setSSLEngineConfig(buildSSLEngineConfigurator(listenerConfig.getTlsConfiguration())); + if (listenerConfig.getTlsConfig().isEnabled()) { + listener.setSecure(true); + listener.setSSLEngineConfig(buildSSLEngineConfigurator(listenerConfig.getTlsConfig())); } + httpServer.addListener(listener); } diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/listeners/ListenerConfig.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/listeners/ListenerConfig.java index 07df6a1..822912f 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/listeners/ListenerConfig.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/listeners/ListenerConfig.java @@ -18,42 +18,45 @@ */ package org.apache.pinot.controller.api.listeners; +import org.apache.pinot.core.transport.TlsConfig; + + /** * Provides configuration settings expected by an Http Server to * setup listeners for http and https protocols. */ public class ListenerConfig { - private final String name; - private final String host; - private final int port; - private final String protocol; - private final TlsConfiguration tlsConfiguration; - - public ListenerConfig(String name, String host, int port, String protocol, TlsConfiguration tlsConfiguration) { - this.name = name; - this.host = host; - this.port = port; - this.protocol = protocol; - this.tlsConfiguration = tlsConfiguration; + private final String _name; + private final String _host; + private final int _port; + private final String _protocol; + private final TlsConfig _tlsConfig; + + public ListenerConfig(String name, String host, int port, String protocol, TlsConfig tlsConfig) { + this._name = name; + this._host = host; + this._port = port; + this._protocol = protocol; + this._tlsConfig = tlsConfig; } public String getName() { - return name; + return _name; } public String getHost() { - return host; + return _host; } public int getPort() { - return port; + return _port; } public String getProtocol() { - return protocol; + return _protocol; } - public TlsConfiguration getTlsConfiguration() { - return tlsConfiguration; + public TlsConfig getTlsConfig() { + return _tlsConfig; } } diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/listeners/TlsConfiguration.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/listeners/TlsConfiguration.java deleted file mode 100644 index d3edde0..0000000 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/listeners/TlsConfiguration.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.apache.pinot.controller.api.listeners; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Holds TLS configuration settings. Used as a vessel to configure Https Listeners. - * - * @author Daniel Lavoie - * @since 0.4.0 - */ -public class TlsConfiguration { - private final String keyStorePath; - private final String keyStorePassword; - private final String trustStorePath; - private final String trustStorePassword; - private final boolean requiresClientAuth; - - public TlsConfiguration(String keyStorePath, String keyStorePassword, String trustStorePath, - String trustStorePassword, boolean requiresClientAuth) { - this.keyStorePath = keyStorePath; - this.keyStorePassword = keyStorePassword; - this.trustStorePath = trustStorePath; - this.trustStorePassword = trustStorePassword; - this.requiresClientAuth = requiresClientAuth; - } - - public String getKeyStorePath() { - return keyStorePath; - } - - public String getKeyStorePassword() { - return keyStorePassword; - } - - public String getTrustStorePath() { - return trustStorePath; - } - - public String getTrustStorePassword() { - return trustStorePassword; - } - - public boolean isRequiresClientAuth() { - return requiresClientAuth; - } -} diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ListenerConfigUtil.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ListenerConfigUtil.java index 398cd78..f33f523 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ListenerConfigUtil.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ListenerConfigUtil.java @@ -22,10 +22,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; - +import org.apache.pinot.common.utils.CommonConstants; import org.apache.pinot.controller.ControllerConf; import org.apache.pinot.controller.api.listeners.ListenerConfig; -import org.apache.pinot.controller.api.listeners.TlsConfiguration; +import org.apache.pinot.core.transport.TlsConfig; +import org.apache.pinot.core.util.TlsUtils; /** @@ -47,7 +48,8 @@ public abstract class ListenerConfigUtil { if (controllerConf.getControllerPort() != null) { listenerConfigs.add( - new ListenerConfig("http", "0.0.0.0", Integer.valueOf(controllerConf.getControllerPort()), "http", null)); + new ListenerConfig("http", "0.0.0.0", Integer.parseInt(controllerConf.getControllerPort()), + "http", new TlsConfig())); } listenerConfigs.addAll(controllerConf.getControllerAccessProtocols().stream() @@ -59,26 +61,13 @@ public abstract class ListenerConfigUtil { return listenerConfigs; } - private static Optional<TlsConfiguration> buildTlsConfiguration(String protocol, ControllerConf controllerConf) { - return Optional.ofNullable(controllerConf.getControllerAccessProtocolProperty(protocol, "tls.keystore.path")) - - .map(keystore -> buildTlsConfiguration(protocol, keystore, controllerConf)); - } - - private static TlsConfiguration buildTlsConfiguration(String protocol, String keystore, - ControllerConf controllerConf) { - return new TlsConfiguration(keystore, - controllerConf.getControllerAccessProtocolProperty(protocol, "tls.keystore.password"), - controllerConf.getControllerAccessProtocolProperty(protocol, "tls.truststore.path"), - controllerConf.getControllerAccessProtocolProperty(protocol, "tls.truststore.password"), Boolean.parseBoolean( - controllerConf.getControllerAccessProtocolProperty(protocol, "tls.requires_client_auth", "false"))); - } - private static ListenerConfig buildListenerConfig(String protocol, ControllerConf controllerConf) { + TlsConfig tlsConfig = TlsUtils.extractTlsConfig(controllerConf, "controller.access.protocols." + protocol); + tlsConfig.setEnabled(CommonConstants.HTTPS_PROTOCOL.equals(protocol)); + return new ListenerConfig(protocol, getHost(controllerConf.getControllerAccessProtocolProperty(protocol, "host", "0.0.0.0")), - getPort(controllerConf.getControllerAccessProtocolProperty(protocol, "port")), protocol, - buildTlsConfiguration(protocol, controllerConf).orElse(null)); + getPort(controllerConf.getControllerAccessProtocolProperty(protocol, "port")), protocol, tlsConfig); } private static String getHost(String configuredHost) { diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/util/ListenerConfigUtilTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/util/ListenerConfigUtilTest.java index bcb4333..c8c745b 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/util/ListenerConfigUtilTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/util/ListenerConfigUtilTest.java @@ -174,7 +174,7 @@ public class ListenerConfigUtilTest { Assert.assertEquals(legacyListener.getHost(), "0.0.0.0"); Assert.assertEquals(legacyListener.getPort(), 9000); Assert.assertEquals(legacyListener.getProtocol(), "http"); - Assert.assertNull(legacyListener.getTlsConfiguration()); + Assert.assertFalse(legacyListener.getTlsConfig().isEnabled()); } private void assertHttpListener(ListenerConfig httpsListener, String host, int port) { @@ -182,7 +182,7 @@ public class ListenerConfigUtilTest { Assert.assertEquals(httpsListener.getHost(), host); Assert.assertEquals(httpsListener.getPort(), port); Assert.assertEquals(httpsListener.getProtocol(), "http"); - Assert.assertNull(httpsListener.getTlsConfiguration()); + Assert.assertFalse(httpsListener.getTlsConfig().isEnabled()); } private void assertHttpsListener(ListenerConfig httpsListener, String host, int port) { @@ -190,12 +190,13 @@ public class ListenerConfigUtilTest { Assert.assertEquals(httpsListener.getHost(), host); Assert.assertEquals(httpsListener.getPort(), port); Assert.assertEquals(httpsListener.getProtocol(), "https"); - Assert.assertNotNull(httpsListener.getTlsConfiguration()); - Assert.assertEquals(httpsListener.getTlsConfiguration().getKeyStorePassword(), "a-password"); - Assert.assertEquals(httpsListener.getTlsConfiguration().getKeyStorePath(), "/some-keystore-path"); - Assert.assertEquals(httpsListener.getTlsConfiguration().isRequiresClientAuth(), true); - Assert.assertEquals(httpsListener.getTlsConfiguration().getTrustStorePassword(), "a-password"); - Assert.assertEquals(httpsListener.getTlsConfiguration().getTrustStorePath(), "/some-truststore-path"); + Assert.assertNotNull(httpsListener.getTlsConfig()); + Assert.assertTrue(httpsListener.getTlsConfig().isEnabled()); + Assert.assertEquals(httpsListener.getTlsConfig().getKeyStorePassword(), "a-password"); + Assert.assertEquals(httpsListener.getTlsConfig().getKeyStorePath(), "/some-keystore-path"); + Assert.assertTrue(httpsListener.getTlsConfig().isClientAuth()); + Assert.assertEquals(httpsListener.getTlsConfig().getTrustStorePassword(), "a-password"); + Assert.assertEquals(httpsListener.getTlsConfig().getTrustStorePath(), "/some-truststore-path"); } private void configureHttpsProperties(ControllerConf controllerConf, String host, int port) { @@ -205,7 +206,7 @@ public class ListenerConfigUtilTest { controllerConf.setProperty("controller.access.protocols.https.port", String.valueOf(port)); controllerConf.setProperty("controller.access.protocols.https.tls.keystore.password", "a-password"); controllerConf.setProperty("controller.access.protocols.https.tls.keystore.path", "/some-keystore-path"); - controllerConf.setProperty("controller.access.protocols.https.tls.requires_client_auth", "true"); + controllerConf.setProperty("controller.access.protocols.https.tls.client.auth", "true"); controllerConf.setProperty("controller.access.protocols.https.tls.truststore.password", "a-password"); controllerConf.setProperty("controller.access.protocols.https.tls.truststore.path", "/some-truststore-path"); } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/QueryRouter.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/QueryRouter.java index 2710a16..6e2e9ae 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/transport/QueryRouter.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/QueryRouter.java @@ -48,12 +48,31 @@ public class QueryRouter { private final ServerChannels _serverChannels; private final ConcurrentHashMap<Long, AsyncQueryResponse> _asyncQueryResponseMap = new ConcurrentHashMap<>(); + /** + * Create an unsecured query router + * + * @param brokerId broker id + * @param brokerMetrics broker metrics + */ public QueryRouter(String brokerId, BrokerMetrics brokerMetrics) { _brokerId = brokerId; _brokerMetrics = brokerMetrics; _serverChannels = new ServerChannels(this, brokerMetrics); } + /** + * Create a query router with TLS config + * + * @param brokerId broker id + * @param brokerMetrics broker metrics + * @param tlsConfig TLS config + */ + public QueryRouter(String brokerId, BrokerMetrics brokerMetrics, TlsConfig tlsConfig) { + _brokerId = brokerId; + _brokerMetrics = brokerMetrics; + _serverChannels = new ServerChannels(this, brokerMetrics, tlsConfig); + } + public AsyncQueryResponse submitQuery(long requestId, String rawTableName, @Nullable BrokerRequest offlineBrokerRequest, @Nullable Map<ServerInstance, List<String>> offlineRoutingTable, @Nullable BrokerRequest realtimeBrokerRequest, @Nullable Map<ServerInstance, List<String>> realtimeRoutingTable, diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/QueryServer.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/QueryServer.java index 6299f94..2c85afa 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/transport/QueryServer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/QueryServer.java @@ -28,9 +28,14 @@ import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.codec.LengthFieldPrepender; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContextBuilder; import java.util.concurrent.TimeUnit; import org.apache.pinot.common.metrics.ServerMetrics; import org.apache.pinot.core.query.scheduler.QueryScheduler; +import org.apache.pinot.core.util.TlsUtils; /** @@ -41,15 +46,36 @@ public class QueryServer { private final int _port; private final QueryScheduler _queryScheduler; private final ServerMetrics _serverMetrics; + private final TlsConfig _tlsConfig; private EventLoopGroup _bossGroup; private EventLoopGroup _workerGroup; private Channel _channel; + /** + * Create an unsecured server instance + * + * @param port bind port + * @param queryScheduler query scheduler + * @param serverMetrics server metrics + */ public QueryServer(int port, QueryScheduler queryScheduler, ServerMetrics serverMetrics) { + this(port, queryScheduler, serverMetrics, null); + } + + /** + * Create a server instance with TLS config + * + * @param port bind port + * @param queryScheduler query scheduler + * @param serverMetrics server metrics + * @param tlsConfig TLS/SSL config + */ + public QueryServer(int port, QueryScheduler queryScheduler, ServerMetrics serverMetrics, TlsConfig tlsConfig) { _port = port; _queryScheduler = queryScheduler; _serverMetrics = serverMetrics; + _tlsConfig = tlsConfig; } public void start() { @@ -62,6 +88,10 @@ public class QueryServer { .childHandler(new ChannelInitializer<SocketChannel>() { @Override protected void initChannel(SocketChannel ch) { + if (_tlsConfig != null && _tlsConfig.isEnabled()) { + attachSSLHandler(ch); + } + ch.pipeline() .addLast(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, Integer.BYTES, 0, Integer.BYTES), new LengthFieldPrepender(Integer.BYTES), @@ -76,6 +106,29 @@ public class QueryServer { } } + private void attachSSLHandler(SocketChannel ch) { + try { + if (_tlsConfig.getKeyStorePath() == null) { + throw new IllegalArgumentException("Must provide key store path for secured server"); + } + + SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(TlsUtils.createKeyManagerFactory(_tlsConfig)); + + if (_tlsConfig.getTrustStorePath() != null) { + sslContextBuilder.trustManager(TlsUtils.createTrustManagerFactory(_tlsConfig)); + } + + if (_tlsConfig.isClientAuth()) { + sslContextBuilder.clientAuth(ClientAuth.REQUIRE); + } + + ch.pipeline().addLast("ssl", sslContextBuilder.build().newHandler(ch.alloc())); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + public void shutDown() { try { _channel.close().sync(); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerChannels.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerChannels.java index 7c05742..e9cd2aa 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerChannels.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerChannels.java @@ -29,6 +29,8 @@ import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.codec.LengthFieldPrepender; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContextBuilder; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import javax.annotation.concurrent.ThreadSafe; @@ -36,6 +38,7 @@ import org.apache.pinot.common.metrics.BrokerGauge; import org.apache.pinot.common.metrics.BrokerMeter; import org.apache.pinot.common.metrics.BrokerMetrics; import org.apache.pinot.common.request.InstanceRequest; +import org.apache.pinot.core.util.TlsUtils; import org.apache.thrift.TSerializer; import org.apache.thrift.protocol.TCompactProtocol; @@ -51,10 +54,29 @@ public class ServerChannels { private final BrokerMetrics _brokerMetrics; private final ConcurrentHashMap<ServerRoutingInstance, ServerChannel> _serverToChannelMap = new ConcurrentHashMap<>(); private final EventLoopGroup _eventLoopGroup = new NioEventLoopGroup(); + private final TlsConfig _tlsConfig; + /** + * Create an unsecured server channel + * + * @param queryRouter query router + * @param brokerMetrics broker metrics + */ public ServerChannels(QueryRouter queryRouter, BrokerMetrics brokerMetrics) { + this(queryRouter, brokerMetrics, null); + } + + /** + * Create a server channel with TLS config + * + * @param queryRouter query router + * @param brokerMetrics broker metrics + * @param tlsConfig TLS/SSL config + */ + public ServerChannels(QueryRouter queryRouter, BrokerMetrics brokerMetrics, TlsConfig tlsConfig) { _queryRouter = queryRouter; _brokerMetrics = brokerMetrics; + _tlsConfig = tlsConfig; } public void sendRequest(ServerRoutingInstance serverRoutingInstance, InstanceRequest instanceRequest) @@ -81,6 +103,10 @@ public class ServerChannels { .handler(new ChannelInitializer<SocketChannel>() { @Override protected void initChannel(SocketChannel ch) { + if (_tlsConfig != null && _tlsConfig.isEnabled()) { + attachSSLHandler(ch); + } + ch.pipeline() .addLast(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, Integer.BYTES, 0, Integer.BYTES), new LengthFieldPrepender(Integer.BYTES), @@ -91,6 +117,25 @@ public class ServerChannels { }); } + private void attachSSLHandler(SocketChannel ch) { + try { + SslContextBuilder sslContextBuilder = SslContextBuilder.forClient(); + + if (_tlsConfig.getKeyStorePath() != null) { + sslContextBuilder.keyManager(TlsUtils.createKeyManagerFactory(_tlsConfig)); + } + + if (_tlsConfig.getTrustStorePath() != null) { + sslContextBuilder.trustManager(TlsUtils.createTrustManagerFactory(_tlsConfig)); + } + + ch.pipeline().addLast("ssl", sslContextBuilder.build().newHandler(ch.alloc())); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + synchronized void sendRequest(InstanceRequest instanceRequest) throws Exception { if (_channel == null || !_channel.isActive()) { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/TlsConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/TlsConfig.java new file mode 100644 index 0000000..0ea7f47 --- /dev/null +++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/TlsConfig.java @@ -0,0 +1,61 @@ +package org.apache.pinot.core.transport; + +/** + * Container object for TLS/SSL configuration of pinot clients and servers (netty, grizzly, etc.) + */ +public class TlsConfig { + private boolean _enabled; + private boolean _clientAuth; + private String _keyStorePath; + private String _keyStorePassword; + private String _trustStorePath; + private String _trustStorePassword; + + public boolean isEnabled() { + return _enabled; + } + + public void setEnabled(boolean enabled) { + _enabled = enabled; + } + + public boolean isClientAuth() { + return _clientAuth; + } + + public void setClientAuth(boolean clientAuth) { + _clientAuth = clientAuth; + } + + public String getKeyStorePath() { + return _keyStorePath; + } + + public void setKeyStorePath(String keyStorePath) { + _keyStorePath = keyStorePath; + } + + public String getKeyStorePassword() { + return _keyStorePassword; + } + + public void setKeyStorePassword(String keyStorePassword) { + _keyStorePassword = keyStorePassword; + } + + public String getTrustStorePath() { + return _trustStorePath; + } + + public void setTrustStorePath(String trustStorePath) { + _trustStorePath = trustStorePath; + } + + public String getTrustStorePassword() { + return _trustStorePassword; + } + + public void setTrustStorePassword(String trustStorePassword) { + _trustStorePassword = trustStorePassword; + } +} diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/TlsUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/util/TlsUtils.java new file mode 100644 index 0000000..f8adc85 --- /dev/null +++ b/pinot-core/src/main/java/org/apache/pinot/core/util/TlsUtils.java @@ -0,0 +1,132 @@ +package org.apache.pinot.core.util; + +import com.google.common.base.Preconditions; +import java.io.FileInputStream; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import org.apache.pinot.core.transport.TlsConfig; +import org.apache.pinot.spi.env.PinotConfiguration; + + +/** + * Utility class for shared TLS configuration logic + */ +public final class TlsUtils { + private static final String TLS_ENABLED = "tls.enabled"; + private static final String TLS_CIENT_AUTH = "tls.client.auth"; + private static final String TLS_KEYSTORE_PATH = "tls.keystore.path"; + private static final String TLS_KEYSTORE_PASSWORD = "tls.keystore.password"; + private static final String TLS_TRUSTSTORE_PATH = "tls.truststore.path"; + private static final String TLS_TRUSTSTORE_PASSWORD = "tls.truststore.password"; + + private TlsUtils() { + // left blank + } + + /** + * Extract a TlsConfig instance from a namespaced set of configuration keys. + * + * @param pinotConfig pinot configuration + * @param prefix namespace prefix + * + * @return TlsConfig instance + */ + public static TlsConfig extractTlsConfig(PinotConfiguration pinotConfig, String prefix) { + TlsConfig tlsConfig = new TlsConfig(); + + tlsConfig.setEnabled(pinotConfig.getProperty(prefix + "." + TLS_ENABLED, false)); + tlsConfig.setClientAuth(pinotConfig.getProperty(prefix + "." + TLS_CIENT_AUTH, false)); + tlsConfig.setKeyStorePath(pinotConfig.getProperty(prefix + "." + TLS_KEYSTORE_PATH)); + tlsConfig.setKeyStorePassword(pinotConfig.getProperty(prefix + "." + TLS_KEYSTORE_PASSWORD)); + tlsConfig.setTrustStorePath(pinotConfig.getProperty(prefix + "." + TLS_TRUSTSTORE_PATH)); + tlsConfig.setTrustStorePassword(pinotConfig.getProperty(prefix + "." + TLS_TRUSTSTORE_PASSWORD)); + + return tlsConfig; + } + + /** + * Create a KeyManagerFactory instance from a given TlsConfig. + * + * @param tlsConfig TLS config + * + * @return KeyManagerFactory + */ + public static KeyManagerFactory createKeyManagerFactory(TlsConfig tlsConfig) { + Preconditions.checkNotNull(tlsConfig.getKeyStorePath(), "key store path is null"); + Preconditions.checkNotNull(tlsConfig.getKeyStorePassword(), "key store password is null"); + + try { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + try (FileInputStream is = new FileInputStream(tlsConfig.getKeyStorePath())) { + keyStore.load(is, tlsConfig.getKeyStorePassword().toCharArray()); + } + + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, tlsConfig.getKeyStorePassword().toCharArray()); + + return keyManagerFactory; + + } catch (Exception e) { + throw new RuntimeException(String.format("Could not create key manager factory '%s'", + tlsConfig.getKeyStorePath()), e); + } + } + + /** + * Create a TrustManagerFactory instance from a given TlsConfig. + * + * @param tlsConfig TLS config + * + * @return TrustManagerFactory + */ + public static TrustManagerFactory createTrustManagerFactory(TlsConfig tlsConfig) { + Preconditions.checkNotNull(tlsConfig.getTrustStorePath(), "trust store path is null"); + Preconditions.checkNotNull(tlsConfig.getTrustStorePassword(), "trust store password is null"); + + try { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + try (FileInputStream is = new FileInputStream(tlsConfig.getTrustStorePath())) { + keyStore.load(is, tlsConfig.getTrustStorePassword().toCharArray()); + } + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + + return trustManagerFactory; + } catch (Exception e) { + throw new RuntimeException(String.format("Could not create trust manager factory '%s'", + tlsConfig.getTrustStorePath()), e); + } + } + + /** + * Installs a default TLS socket factory for all HttpsURLConnection instances based on a given TlsConfig (1 or 2-way) + * + * @param tlsConfig TLS config + */ + public static void installDefaultSSLSocketFactory(TlsConfig tlsConfig) { + KeyManager[] keyManagers = null; + if (tlsConfig.getKeyStorePath() != null) { + keyManagers = createKeyManagerFactory(tlsConfig).getKeyManagers(); + } + + TrustManager[] trustManagers = null; + if (tlsConfig.getTrustStorePath() != null) { + trustManagers = createTrustManagerFactory(tlsConfig).getTrustManagers(); + } + + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(keyManagers, trustManagers, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (GeneralSecurityException ignore) { + // ignore + } + } +} diff --git a/pinot-server/src/main/java/org/apache/pinot/server/conf/ServerConf.java b/pinot-server/src/main/java/org/apache/pinot/server/conf/ServerConf.java index baf916f..0f5e66a 100644 --- a/pinot-server/src/main/java/org/apache/pinot/server/conf/ServerConf.java +++ b/pinot-server/src/main/java/org/apache/pinot/server/conf/ServerConf.java @@ -121,4 +121,8 @@ public class ServerConf { public String getMetricsPrefix() { return _serverConf.getProperty(PINOT_SERVER_METRICS_PREFIX, Server.DEFAULT_METRICS_PREFIX); } + + public PinotConfiguration getPinotConfig() { + return _serverConf; + } } diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/ServerInstance.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/ServerInstance.java index 462a304..2c0188b 100644 --- a/pinot-server/src/main/java/org/apache/pinot/server/starter/ServerInstance.java +++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/ServerInstance.java @@ -34,7 +34,9 @@ import org.apache.pinot.core.query.executor.QueryExecutor; import org.apache.pinot.core.query.scheduler.QueryScheduler; import org.apache.pinot.core.query.scheduler.QuerySchedulerFactory; import org.apache.pinot.core.transport.QueryServer; +import org.apache.pinot.core.transport.TlsConfig; import org.apache.pinot.core.transport.grpc.GrpcQueryServer; +import org.apache.pinot.core.util.TlsUtils; import org.apache.pinot.server.conf.ServerConf; import org.apache.pinot.spi.env.PinotConfiguration; import org.slf4j.Logger; @@ -90,10 +92,15 @@ public class ServerInstance { QuerySchedulerFactory.create(serverConf.getSchedulerConfig(), _queryExecutor, _serverMetrics, _latestQueryTime); int nettyPort = serverConf.getNettyPort(); - LOGGER.info("Initializing Netty query server on port: {}", nettyPort); - _nettyQueryServer = new QueryServer(nettyPort, _queryScheduler, _serverMetrics); + TlsConfig tlsConfig = TlsUtils.extractTlsConfig(serverConf.getPinotConfig(), "pinot.server.netty"); + LOGGER.info("Initializing Netty query server on port: {} with tls: {}", nettyPort, tlsConfig.isEnabled()); + _nettyQueryServer = new QueryServer(nettyPort, _queryScheduler, _serverMetrics, tlsConfig); if (serverConf.isEnableGrpcServer()) { + if (tlsConfig.isEnabled()) { + LOGGER.warn("gRPC query server does not support TLS yet"); + } + int grpcPort = serverConf.getGrpcPort(); LOGGER.info("Initializing gRPC query server on port: {}", grpcPort); _grpcQueryServer = new GrpcQueryServer(grpcPort, _queryExecutor, _serverMetrics); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
