This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 23a81d07b5 Add TLS configuration to JDBC driver (#8578)
23a81d07b5 is described below
commit 23a81d07b52dad6181b696562e07cdfa0932d191
Author: Kartik Khare <[email protected]>
AuthorDate: Wed May 11 20:09:23 2022 +0530
Add TLS configuration to JDBC driver (#8578)
* Add TLS configuration to JDBC driver
* Add TLS tests for JDBC
* use TLSUtil properties parser instead of JDBC utils
Co-authored-by: Kartik Khare <[email protected]>
---
pinot-clients/pinot-jdbc-client/pom.xml | 4 ++
.../java/org/apache/pinot/client/PinotDriver.java | 35 ++++++++++---
.../controller/PinotControllerTransport.java | 31 ++++++++++--
.../PinotControllerTransportFactory.java | 59 ++++++++++++++++++++++
.../org/apache/pinot/client/utils/DriverUtils.java | 19 +++++--
pinot-integration-tests/pom.xml | 4 ++
.../integration/tests/TlsIntegrationTest.java | 51 +++++++++++++++++++
pom.xml | 5 ++
8 files changed, 195 insertions(+), 13 deletions(-)
diff --git a/pinot-clients/pinot-jdbc-client/pom.xml
b/pinot-clients/pinot-jdbc-client/pom.xml
index 1bb924a789..b61deab966 100644
--- a/pinot-clients/pinot-jdbc-client/pom.xml
+++ b/pinot-clients/pinot-jdbc-client/pom.xml
@@ -65,6 +65,10 @@
<artifactId>pinot-java-client</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-common</artifactId>
+ </dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
diff --git
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotDriver.java
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotDriver.java
index 8dfc88b77e..4055d788d3 100644
---
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotDriver.java
+++
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotDriver.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.client;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import java.net.URI;
import java.sql.Connection;
@@ -32,9 +33,13 @@ import java.util.Map;
import java.util.Properties;
import java.util.logging.Logger;
import java.util.stream.Collectors;
+import javax.net.ssl.SSLContext;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.client.controller.PinotControllerTransport;
+import org.apache.pinot.client.controller.PinotControllerTransportFactory;
import org.apache.pinot.client.utils.DriverUtils;
+import org.apache.pinot.common.utils.TlsUtils;
+import org.apache.pinot.spi.utils.CommonConstants;
import org.slf4j.LoggerFactory;
@@ -45,6 +50,14 @@ public class PinotDriver implements Driver {
public static final String DEFAULT_TENANT = "DefaultTenant";
public static final String INFO_SCHEME = "scheme";
public static final String INFO_HEADERS = "headers";
+ private SSLContext _sslContext = null;
+
+ public PinotDriver() { }
+
+ @VisibleForTesting
+ public PinotDriver(SSLContext sslContext) {
+ _sslContext = sslContext;
+ }
@Override
public Connection connect(String url, Properties info)
@@ -52,9 +65,20 @@ public class PinotDriver implements Driver {
try {
LOGGER.info("Initiating connection to database for url: " + url);
JsonAsyncHttpPinotClientTransportFactory factory = new
JsonAsyncHttpPinotClientTransportFactory();
+ PinotControllerTransportFactory pinotControllerTransportFactory = new
PinotControllerTransportFactory();
- if (info.contains(INFO_SCHEME)) {
+ if (info.containsKey(INFO_SCHEME)) {
factory.setScheme(info.getProperty(INFO_SCHEME));
+
pinotControllerTransportFactory.setScheme(info.getProperty(INFO_SCHEME));
+ if
(info.getProperty(INFO_SCHEME).contentEquals(CommonConstants.HTTPS_PROTOCOL)) {
+ if (_sslContext == null) {
+
factory.setSslContext(DriverUtils.getSSLContextFromJDBCProps(info));
+
pinotControllerTransportFactory.setSslContext(TlsUtils.getSslContext());
+ } else {
+ factory.setSslContext(_sslContext);
+ pinotControllerTransportFactory.setSslContext(_sslContext);
+ }
+ }
}
Map<String, String> headers =
@@ -62,18 +86,17 @@ public class PinotDriver implements Driver {
entry -> Pair
.of(entry.getKey().toString().substring(INFO_HEADERS.length() + 1),
entry.getValue().toString()))
.collect(Collectors.toMap(Pair::getKey, Pair::getValue));
+
if (!headers.isEmpty()) {
factory.setHeaders(headers);
+ pinotControllerTransportFactory.setHeaders(headers);
}
PinotClientTransport pinotClientTransport = factory.buildTransport();
+ PinotControllerTransport pinotControllerTransport =
pinotControllerTransportFactory.buildTransport();
String controllerUrl = DriverUtils.getControllerFromURL(url);
String tenant = info.getProperty(INFO_TENANT, DEFAULT_TENANT);
- if (!headers.isEmpty()) {
- PinotControllerTransport pinotControllerTransport = new
PinotControllerTransport(headers);
- return new PinotConnection(info, controllerUrl, pinotClientTransport,
tenant, pinotControllerTransport);
- }
- return new PinotConnection(info, controllerUrl, pinotClientTransport,
tenant);
+ return new PinotConnection(info, controllerUrl, pinotClientTransport,
tenant, pinotControllerTransport);
} catch (Exception e) {
throw new SQLException(String.format("Failed to connect to url : %s",
url), e);
}
diff --git
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/PinotControllerTransport.java
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/PinotControllerTransport.java
index ac6860ce46..d0df4e2116 100644
---
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/PinotControllerTransport.java
+++
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/PinotControllerTransport.java
@@ -18,16 +18,23 @@
*/
package org.apache.pinot.client.controller;
+import io.netty.handler.ssl.ClientAuth;
+import io.netty.handler.ssl.JdkSslContext;
import java.io.IOException;
+import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import javax.annotation.Nullable;
+import javax.net.ssl.SSLContext;
import org.apache.pinot.client.PinotClientException;
import
org.apache.pinot.client.controller.response.ControllerTenantBrokerResponse;
import org.apache.pinot.client.controller.response.SchemaResponse;
import org.apache.pinot.client.controller.response.TableResponse;
+import org.apache.pinot.spi.utils.CommonConstants;
import org.asynchttpclient.AsyncHttpClient;
import org.asynchttpclient.BoundRequestBuilder;
+import org.asynchttpclient.DefaultAsyncHttpClientConfig;
import org.asynchttpclient.Dsl;
import org.asynchttpclient.Response;
import org.slf4j.Logger;
@@ -38,19 +45,35 @@ public class PinotControllerTransport {
private static final Logger LOGGER =
LoggerFactory.getLogger(PinotControllerTransport.class);
- AsyncHttpClient _httpClient = Dsl.asyncHttpClient();
Map<String, String> _headers;
+ private final String _scheme;
+ private final AsyncHttpClient _httpClient;
+
public PinotControllerTransport() {
+ this(Collections.emptyMap(), CommonConstants.HTTP_PROTOCOL, null);
}
public PinotControllerTransport(Map<String, String> headers) {
+ this(headers, CommonConstants.HTTP_PROTOCOL, null);
+ }
+
+ public PinotControllerTransport(Map<String, String> headers, String scheme,
+ @Nullable SSLContext sslContext) {
_headers = headers;
+ _scheme = scheme;
+
+ DefaultAsyncHttpClientConfig.Builder builder = Dsl.config();
+ if (sslContext != null) {
+ builder.setSslContext(new JdkSslContext(sslContext, true,
ClientAuth.OPTIONAL));
+ }
+
+ _httpClient = Dsl.asyncHttpClient(builder.build());
}
public TableResponse getAllTables(String controllerAddress) {
try {
- String url = "http://"; + controllerAddress + "/tables";
+ String url = _scheme + "://" + controllerAddress + "/tables";
BoundRequestBuilder requestBuilder = _httpClient.prepareGet(url);
if (_headers != null) {
_headers.forEach((k, v) -> requestBuilder.addHeader(k, v));
@@ -68,7 +91,7 @@ public class PinotControllerTransport {
public SchemaResponse getTableSchema(String table, String controllerAddress)
{
try {
- String url = "http://"; + controllerAddress + "/tables/" + table +
"/schema";
+ String url = _scheme + "://" + controllerAddress + "/tables/" + table +
"/schema";
BoundRequestBuilder requestBuilder = _httpClient.prepareGet(url);
if (_headers != null) {
_headers.forEach((k, v) -> requestBuilder.addHeader(k, v));
@@ -86,7 +109,7 @@ public class PinotControllerTransport {
public ControllerTenantBrokerResponse getBrokersFromController(String
controllerAddress, String tenant) {
try {
- String url = "http://"; + controllerAddress + "/v2/brokers/tenants/" +
tenant;
+ String url = _scheme + "://" + controllerAddress +
"/v2/brokers/tenants/" + tenant;
BoundRequestBuilder requestBuilder = _httpClient.prepareGet(url);
if (_headers != null) {
_headers.forEach((k, v) -> requestBuilder.addHeader(k, v));
diff --git
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/PinotControllerTransportFactory.java
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/PinotControllerTransportFactory.java
new file mode 100644
index 0000000000..ead3ee637f
--- /dev/null
+++
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/PinotControllerTransportFactory.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.client.controller;
+
+import java.util.HashMap;
+import java.util.Map;
+import javax.net.ssl.SSLContext;
+import org.apache.pinot.spi.utils.CommonConstants;
+
+
+public class PinotControllerTransportFactory {
+ private Map<String, String> _headers = new HashMap<>();
+ private String _scheme = CommonConstants.HTTP_PROTOCOL;
+ private SSLContext _sslContext = null;
+
+ public PinotControllerTransport buildTransport() {
+ return new PinotControllerTransport(_headers, _scheme, _sslContext);
+ }
+
+ public Map<String, String> getHeaders() {
+ return _headers;
+ }
+
+ public void setHeaders(Map<String, String> headers) {
+ _headers = headers;
+ }
+
+ public String getScheme() {
+ return _scheme;
+ }
+
+ public void setScheme(String scheme) {
+ _scheme = scheme;
+ }
+
+ public SSLContext getSslContext() {
+ return _sslContext;
+ }
+
+ public void setSslContext(SSLContext sslContext) {
+ _sslContext = sslContext;
+ }
+}
diff --git
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/DriverUtils.java
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/DriverUtils.java
index 3f68460aaa..710a9d1725 100644
---
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/DriverUtils.java
+++
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/DriverUtils.java
@@ -24,8 +24,14 @@ import java.sql.Timestamp;
import java.sql.Types;
import java.util.Collections;
import java.util.List;
+import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.net.ssl.SSLContext;
+import org.apache.commons.configuration.MapConfiguration;
+import org.apache.pinot.common.config.TlsConfig;
+import org.apache.pinot.common.utils.TlsUtils;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,14 +40,21 @@ public class DriverUtils {
public static final String SCHEME = "jdbc";
public static final String DRIVER = "pinot";
public static final Logger LOG = LoggerFactory.getLogger(DriverUtils.class);
- public static final String QUERY_SEPERATOR = "&";
- public static final String PARAM_SEPERATOR = "=";
- public static final String CONTROLLER = "controller";
private static final String LIMIT_STATEMENT_REGEX = "\\s(limit)\\s";
+ // SSL Properties
+ public static final String PINOT_JDBC_TLS_PREFIX = "pinot.jdbc.tls";
+
private DriverUtils() {
}
+ public static SSLContext getSSLContextFromJDBCProps(Properties properties) {
+ TlsConfig tlsConfig = TlsUtils.extractTlsConfig(
+ new PinotConfiguration(new MapConfiguration(properties)),
PINOT_JDBC_TLS_PREFIX);
+ TlsUtils.installDefaultSSLSocketFactory(tlsConfig);
+ return TlsUtils.getSslContext();
+ }
+
public static List<String> getBrokersFromURL(String url) {
if (url.toLowerCase().startsWith("jdbc:")) {
url = url.substring(5);
diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml
index a688371d59..37e4375ea0 100644
--- a/pinot-integration-tests/pom.xml
+++ b/pinot-integration-tests/pom.xml
@@ -193,6 +193,10 @@
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-java-client</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-jdbc-client</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-server</artifactId>
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
index bbc86c2e5d..22ed5fc4ac 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
@@ -24,10 +24,13 @@ import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URL;
+import java.sql.ResultSet;
+import java.sql.Statement;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.stream.Collectors;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.io.FileUtils;
@@ -44,6 +47,7 @@ import org.apache.http.ssl.SSLContextBuilder;
import org.apache.pinot.client.Connection;
import org.apache.pinot.client.ConnectionFactory;
import org.apache.pinot.client.JsonAsyncHttpPinotClientTransportFactory;
+import org.apache.pinot.client.PinotDriver;
import org.apache.pinot.client.ResultSetGroup;
import org.apache.pinot.common.helix.ExtraInstanceConfig;
import org.apache.pinot.common.utils.SimpleHttpResponse;
@@ -496,6 +500,53 @@ public class TlsIntegrationTest extends
BaseClusterIntegrationTest {
}
}
+ @Test
+ public void testJDBCClient()
+ throws Exception {
+ String query = "SELECT count(*) FROM " + getTableName();
+ java.sql.Connection connection =
getValidJDBCConnection(DEFAULT_CONTROLLER_PORT);
+ Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(query);
+ resultSet.first();
+ Assert.assertTrue(resultSet.getLong(1) > 0);
+
+ try {
+ java.sql.Connection invalidConnection =
getInValidJDBCConnection(DEFAULT_CONTROLLER_PORT);
+ statement = invalidConnection.createStatement();
+ resultSet = statement.executeQuery(query);
+ Assert.fail("Should not allow queries with invalid TLS configuration");
+ } catch (Exception e) {
+ // this should fail
+ }
+ }
+
+ private java.sql.Connection getValidJDBCConnection(int controllerPort)
throws Exception {
+ SSLContextBuilder sslContextBuilder = SSLContextBuilder.create();
+ sslContextBuilder.setKeyStoreType(PKCS_12);
+ sslContextBuilder.loadKeyMaterial(_tlsStorePKCS12, PASSWORD_CHAR,
PASSWORD_CHAR);
+ sslContextBuilder.loadTrustMaterial(_tlsStorePKCS12, PASSWORD_CHAR);
+
+ PinotDriver pinotDriver = new PinotDriver(sslContextBuilder.build());
+ Properties jdbcProps = new Properties();
+ jdbcProps.setProperty(PinotDriver.INFO_SCHEME,
CommonConstants.HTTPS_PROTOCOL);
+ jdbcProps.setProperty(PinotDriver.INFO_HEADERS + "." +
CLIENT_HEADER.getName(), CLIENT_HEADER.getValue());
+ return pinotDriver.connect("jdbc:pinot://localhost:" + controllerPort,
jdbcProps);
+ }
+
+ private java.sql.Connection getInValidJDBCConnection(int controllerPort)
throws Exception {
+ SSLContextBuilder sslContextBuilder = SSLContextBuilder.create();
+ sslContextBuilder.setKeyStoreType(PKCS_12);
+ sslContextBuilder.loadKeyMaterial(_tlsStoreEmptyPKCS12, PASSWORD_CHAR,
PASSWORD_CHAR);
+ sslContextBuilder.loadTrustMaterial(_tlsStorePKCS12, PASSWORD_CHAR);
+
+ PinotDriver pinotDriver = new PinotDriver(sslContextBuilder.build());
+ Properties jdbcProps = new Properties();
+
+ jdbcProps.setProperty(PinotDriver.INFO_SCHEME,
CommonConstants.HTTPS_PROTOCOL);
+ jdbcProps.setProperty(PinotDriver.INFO_HEADERS + "." +
CLIENT_HEADER.getName(), CLIENT_HEADER.getValue());
+ return pinotDriver.connect("jdbc:pinot://localhost:" + controllerPort,
jdbcProps);
+ }
+
private static CloseableHttpClient makeClient(String keyStoreType, URL
keyStoreUrl, URL trustStoreUrl) {
try {
SSLContextBuilder sslContextBuilder = SSLContextBuilder.create();
diff --git a/pom.xml b/pom.xml
index 567c03da6c..2d0114f42a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -322,6 +322,11 @@
<artifactId>pinot-java-client</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-jdbc-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-core</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
