This is an automated email from the ASF dual-hosted git repository.
tison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zookeeper.git
The following commit(s) were added to refs/heads/master by this push:
new c44cb3798 ZOOKEEPER-4798: Secure prometheus support (#2127)
c44cb3798 is described below
commit c44cb3798245bd3de0157be7997d7125fde19193
Author: puru <[email protected]>
AuthorDate: Thu Aug 1 20:47:46 2024 -0700
ZOOKEEPER-4798: Secure prometheus support (#2127)
Co-authored-by: purshotam shah <[email protected]>
Co-authored-by: tison <[email protected]>
---
.../prometheus/PrometheusMetricsProvider.java | 106 +++++++++++++-
.../PrometheusHttpsMetricsProviderTest.java | 161 +++++++++++++++++++++
.../PrometheusMetricsProviderConfigTest.java | 47 +++++-
.../src/test/resources/data/ssl/README.md | 5 +
.../test/resources/data/ssl/client_keystore.jks | Bin 0 -> 2246 bytes
.../test/resources/data/ssl/client_truststore.jks | Bin 0 -> 986 bytes
.../test/resources/data/ssl/server_keystore.jks | Bin 0 -> 2276 bytes
.../test/resources/data/ssl/server_truststore.jks | Bin 0 -> 958 bytes
8 files changed, 313 insertions(+), 6 deletions(-)
diff --git
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/main/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProvider.java
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/main/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProvider.java
index 574eaa2ce..cae081f26 100644
---
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/main/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProvider.java
+++
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/main/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProvider.java
@@ -55,9 +55,12 @@ import org.apache.zookeeper.server.RateLogger;
import org.eclipse.jetty.security.ConstraintMapping;
import org.eclipse.jetty.security.ConstraintSecurityHandler;
import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.security.Constraint;
+import org.eclipse.jetty.util.ssl.KeyStoreScanner;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,7 +104,8 @@ public class PrometheusMetricsProvider implements
MetricsProvider {
private final CollectorRegistry collectorRegistry =
CollectorRegistry.defaultRegistry;
private final RateLogger rateLogger = new RateLogger(LOG, 60 * 1000);
private String host = "0.0.0.0";
- private int port = 7000;
+ private int httpPort = -1;
+ private int httpsPort = -1;
private boolean exportJvmInfo = true;
private Server server;
private final MetricsServletImpl servlet = new MetricsServletImpl();
@@ -111,11 +115,48 @@ public class PrometheusMetricsProvider implements
MetricsProvider {
private long workerShutdownTimeoutMs = 1000;
private Optional<ExecutorService> executorOptional = Optional.empty();
+ // Constants for SSL configuration
+ public static final int SCAN_INTERVAL = 60 * 10; // 10 minutes
+ public static final String SSL_KEYSTORE_LOCATION = "ssl.keyStore.location";
+ public static final String SSL_KEYSTORE_PASSWORD = "ssl.keyStore.password";
+ public static final String SSL_KEYSTORE_TYPE = "ssl.keyStore.type";
+ public static final String SSL_TRUSTSTORE_LOCATION =
"ssl.trustStore.location";
+ public static final String SSL_TRUSTSTORE_PASSWORD =
"ssl.trustStore.password";
+ public static final String SSL_TRUSTSTORE_TYPE = "ssl.trustStore.type";
+ public static final String SSL_X509_CN = "ssl.x509.cn";
+ public static final String SSL_X509_REGEX_CN = "ssl.x509.cn.regex";
+ public static final String SSL_NEED_CLIENT_AUTH = "ssl.need.client.auth";
+ public static final String SSL_WANT_CLIENT_AUTH = "ssl.want.client.auth";
+
+ private String keyStorePath;
+ private String keyStorePassword;
+ private String keyStoreType;
+ private String trustStorePath;
+ private String trustStorePassword;
+ private String trustStoreType;
+ private boolean needClientAuth = true;
+ private boolean wantClientAuth = true;
+
@Override
public void configure(Properties configuration) throws
MetricsProviderLifeCycleException {
LOG.info("Initializing metrics, configuration: {}", configuration);
this.host = configuration.getProperty("httpHost", "0.0.0.0");
- this.port = Integer.parseInt(configuration.getProperty("httpPort",
"7000"));
+ if (configuration.containsKey("httpsPort")) {
+ this.httpsPort =
Integer.parseInt(configuration.getProperty("httpsPort"));
+ this.keyStorePath =
configuration.getProperty(SSL_KEYSTORE_LOCATION);
+ this.keyStorePassword =
configuration.getProperty(SSL_KEYSTORE_PASSWORD);
+ this.keyStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE);
+ this.trustStorePath =
configuration.getProperty(SSL_TRUSTSTORE_LOCATION);
+ this.trustStorePassword =
configuration.getProperty(SSL_TRUSTSTORE_PASSWORD);
+ this.trustStoreType =
configuration.getProperty(SSL_TRUSTSTORE_TYPE);
+ this.needClientAuth =
Boolean.parseBoolean(configuration.getProperty(SSL_NEED_CLIENT_AUTH, "true"));
+ this.wantClientAuth =
Boolean.parseBoolean(configuration.getProperty(SSL_WANT_CLIENT_AUTH, "true"));
+ //check if httpPort is also configured
+ this.httpPort =
Integer.parseInt(configuration.getProperty("httpPort", "-1"));
+ } else {
+ // Use the default HTTP port (7000) or the configured port if
HTTPS is not set.
+ this.httpPort =
Integer.parseInt(configuration.getProperty("httpPort", "7000"));
+ }
this.exportJvmInfo =
Boolean.parseBoolean(configuration.getProperty("exportJvmInfo", "true"));
this.numWorkerThreads = Integer.parseInt(
configuration.getProperty(NUM_WORKER_THREADS, "1"));
@@ -129,12 +170,29 @@ public class PrometheusMetricsProvider implements
MetricsProvider {
public void start() throws MetricsProviderLifeCycleException {
this.executorOptional = createExecutor();
try {
- LOG.info("Starting /metrics HTTP endpoint at host: {}, port: {},
exportJvmInfo: {}",
- host, port, exportJvmInfo);
+ LOG.info("Starting /metrics {} endpoint at HTTP port: {}, HTTPS
port: {}, exportJvmInfo: {}",
+ httpPort > 0 ? httpPort : "disabled",
+ httpsPort > 0 ? httpsPort : "disabled",
+ exportJvmInfo);
if (exportJvmInfo) {
DefaultExports.initialize();
}
- server = new Server(new InetSocketAddress(host, port));
+ if (httpPort == -1) {
+ server = new Server();
+ } else {
+ server = new Server(new InetSocketAddress(host, httpPort));
+ }
+ if (httpsPort != -1) {
+ SslContextFactory sslServerContextFactory = new
SslContextFactory.Server();
+ configureSslContextFactory(sslServerContextFactory);
+ KeyStoreScanner keystoreScanner = new
KeyStoreScanner(sslServerContextFactory);
+ keystoreScanner.setScanInterval(SCAN_INTERVAL);
+ server.addBean(keystoreScanner);
+ ServerConnector connector = new ServerConnector(server,
sslServerContextFactory);
+ connector.setPort(httpsPort);
+ connector.setHost(host);
+ server.addConnector(connector);
+ }
ServletContextHandler context = new ServletContextHandler();
context.setContextPath("/");
constrainTraceMethod(context);
@@ -156,6 +214,44 @@ public class PrometheusMetricsProvider implements
MetricsProvider {
}
}
+ @SuppressWarnings("deprecation")
+ private void configureSslContextFactory(SslContextFactory
sslServerContextFactory) {
+ if (keyStorePath != null) {
+ sslServerContextFactory.setKeyStorePath(keyStorePath);
+ } else {
+ LOG.error("KeyStore configuration is incomplete keyStorePath: {}",
keyStorePath);
+ throw new IllegalStateException("KeyStore configuration is
incomplete keyStorePath: " + keyStorePath);
+ }
+ if (keyStorePassword != null) {
+ sslServerContextFactory.setKeyStorePassword(keyStorePassword);
+ } else {
+ LOG.error("keyStorePassword configuration is incomplete ");
+ throw new IllegalStateException("keyStorePassword configuration is
incomplete ");
+ }
+ if (keyStoreType != null) {
+ sslServerContextFactory.setKeyStoreType(keyStoreType);
+ }
+ if (trustStorePath != null) {
+ sslServerContextFactory.setTrustStorePath(trustStorePath);
+ } else {
+ LOG.error("TrustStore configuration is incomplete trustStorePath:
{}", trustStorePath);
+ throw new IllegalStateException("TrustStore configuration is
incomplete trustStorePath: " + trustStorePath);
+ }
+ if (trustStorePassword != null) {
+ sslServerContextFactory.setTrustStorePassword(trustStorePassword);
+ } else {
+ LOG.error("trustStorePassword configuration is incomplete");
+ throw new IllegalStateException("trustStorePassword configuration
is incomplete");
+ }
+ if (trustStoreType != null) {
+ sslServerContextFactory.setTrustStoreType(trustStoreType);
+ }
+ sslServerContextFactory
+ .setNeedClientAuth(needClientAuth);
+ sslServerContextFactory
+ .setWantClientAuth(wantClientAuth);
+ }
+
// for tests
MetricsServletImpl getServlet() {
return servlet;
diff --git
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusHttpsMetricsProviderTest.java
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusHttpsMetricsProviderTest.java
new file mode 100644
index 000000000..a27e9084d
--- /dev/null
+++
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusHttpsMetricsProviderTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.metrics.prometheus;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.KeyStore;
+import java.util.Properties;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import org.apache.zookeeper.metrics.Counter;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests about Prometheus Metrics Provider. Please note that we are not testing
+ * Prometheus but only our integration.
+ */
+public class PrometheusHttpsMetricsProviderTest extends
PrometheusMetricsTestBase {
+
+ private PrometheusMetricsProvider provider;
+ private String httpHost = "127.0.0.1";
+ private int httpsPort = 4443;
+ private int httpPort = 4000;
+ private String testDataPath = System.getProperty("test.data.dir",
"src/test/resources/data");
+
+ public void initializeProviderWithCustomConfig(Properties
inputConfiguration) throws Exception {
+ provider = new PrometheusMetricsProvider();
+ Properties configuration = new Properties();
+ configuration.setProperty("httpHost", httpHost);
+ configuration.setProperty("exportJvmInfo", "false");
+ configuration.setProperty("ssl.keyStore.location", testDataPath +
"/ssl/server_keystore.jks");
+ configuration.setProperty("ssl.keyStore.password", "testpass");
+ configuration.setProperty("ssl.trustStore.location", testDataPath +
"/ssl/server_truststore.jks");
+ configuration.setProperty("ssl.trustStore.password", "testpass");
+ configuration.putAll(inputConfiguration);
+ provider.configure(configuration);
+ provider.start();
+ }
+
+ @AfterEach
+ public void tearDown() {
+ if (provider != null) {
+ provider.stop();
+ }
+ }
+
+ @Test
+ void testHttpResponce() throws Exception {
+ Properties configuration = new Properties();
+ configuration.setProperty("httpPort", String.valueOf(httpPort));
+ initializeProviderWithCustomConfig(configuration);
+ simulateMetricIncrement();
+ validateMetricResponse(callHttpServlet("http://"; + httpHost + ":" +
httpPort + "/metrics"));
+ }
+
+ @Test
+ void testHttpsResponse() throws Exception {
+ Properties configuration = new Properties();
+ configuration.setProperty("httpsPort", String.valueOf(httpsPort));
+ initializeProviderWithCustomConfig(configuration);
+ simulateMetricIncrement();
+ validateMetricResponse(callHttpsServlet("https://"; + httpHost + ":" +
httpsPort + "/metrics"));
+ }
+
+ @Test
+ void testHttpAndHttpsResponce() throws Exception {
+ Properties configuration = new Properties();
+ configuration.setProperty("httpsPort", String.valueOf(httpsPort));
+ configuration.setProperty("httpPort", String.valueOf(httpPort));
+ initializeProviderWithCustomConfig(configuration);
+ simulateMetricIncrement();
+ validateMetricResponse(callHttpServlet("http://"; + httpHost + ":" +
httpPort + "/metrics"));
+ validateMetricResponse(callHttpsServlet("https://"; + httpHost + ":" +
httpsPort + "/metrics"));
+ }
+
+ private String callHttpsServlet(String urlString) throws Exception {
+ // Load and configure the SSL context from the keystore and truststore
+ KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+ try (FileInputStream keystoreStream = new FileInputStream(testDataPath
+ "/ssl/client_keystore.jks")) {
+ keyStore.load(keystoreStream, "testpass".toCharArray());
+ }
+
+ KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
+ try (FileInputStream trustStoreStream = new
FileInputStream(testDataPath + "/ssl/client_truststore.jks")) {
+ trustStore.load(trustStoreStream, "testpass".toCharArray());
+ }
+
+ SSLContext sslContext = SSLContext.getInstance("TLS");
+ KeyManagerFactory keyManagerFactory =
KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ keyManagerFactory.init(keyStore, "testpass".toCharArray());
+ TrustManagerFactory trustManagerFactory = TrustManagerFactory
+ .getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ trustManagerFactory.init(trustStore);
+ sslContext.init(keyManagerFactory.getKeyManagers(),
trustManagerFactory.getTrustManagers(),
+ new java.security.SecureRandom());
+
+
HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
+ URL url = new URL(urlString);
+ HttpsURLConnection connection = (HttpsURLConnection)
url.openConnection();
+ connection.setRequestMethod("GET");
+
+ return readResponse(connection);
+ }
+
+ private String callHttpServlet(String urlString) throws IOException {
+ URL url = new URL(urlString);
+ HttpURLConnection connection = (HttpURLConnection)
url.openConnection();
+ connection.setRequestMethod("GET");
+ return readResponse(connection);
+ }
+
+ private String readResponse(HttpURLConnection connection) throws
IOException {
+ int status = connection.getResponseCode();
+ try (BufferedReader reader = new BufferedReader(
+ new InputStreamReader(status > 299 ?
connection.getErrorStream() : connection.getInputStream()))) {
+ StringBuilder content = new StringBuilder();
+ String inputLine;
+ while ((inputLine = reader.readLine()) != null) {
+ content.append(inputLine).append("\n");
+ }
+ return content.toString().trim();
+ } finally {
+ connection.disconnect();
+ }
+ }
+
+ public void simulateMetricIncrement() {
+ Counter counter = provider.getRootContext().getCounter("cc");
+ counter.add(10);
+ }
+
+ private void validateMetricResponse(String response) throws IOException {
+ assertThat(response, containsString("# TYPE cc counter"));
+ assertThat(response, containsString("cc 10.0"));
+ }
+}
\ No newline at end of file
diff --git
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProviderConfigTest.java
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProviderConfigTest.java
index 64989889b..6ff0da5e2 100644
---
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProviderConfigTest.java
+++
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProviderConfigTest.java
@@ -23,7 +23,6 @@ import java.util.Properties;
import org.apache.zookeeper.metrics.MetricsProviderLifeCycleException;
import org.junit.jupiter.api.Test;
-
public class PrometheusMetricsProviderConfigTest extends
PrometheusMetricsTestBase {
@Test
@@ -59,4 +58,50 @@ public class PrometheusMetricsProviderConfigTest extends
PrometheusMetricsTestBa
provider.start();
}
+ @Test
+ public void testValidSslConfig() throws MetricsProviderLifeCycleException {
+ PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
+ Properties configuration = new Properties();
+ String testDataPath = System.getProperty("test.data.dir",
"src/test/resources/data");
+ configuration.setProperty("httpHost", "127.0.0.1");
+ configuration.setProperty("httpsPort", "50511");
+ configuration.setProperty("ssl.keyStore.location", testDataPath +
"/ssl/server_keystore.jks");
+ configuration.setProperty("ssl.keyStore.password", "testpass");
+ configuration.setProperty("ssl.trustStore.location", testDataPath +
"/ssl/server_truststore.jks");
+ configuration.setProperty("ssl.trustStore.password", "testpass");
+ provider.configure(configuration);
+ provider.start();
+ }
+
+ @Test
+ public void testValidHttpsAndHttpConfig() throws
MetricsProviderLifeCycleException {
+ PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
+ Properties configuration = new Properties();
+ String testDataPath = System.getProperty("test.data.dir",
"src/test/resources/data");
+ configuration.setProperty("httpPort", "50512");
+ configuration.setProperty("httpsPort", "50513");
+ configuration.setProperty("ssl.keyStore.location", testDataPath +
"/ssl/server_keystore.jks");
+ configuration.setProperty("ssl.keyStore.password", "testpass");
+ configuration.setProperty("ssl.trustStore.location", testDataPath +
"/ssl/server_truststore.jks");
+ configuration.setProperty("ssl.trustStore.password", "testpass");
+ provider.configure(configuration);
+ provider.start();
+ }
+
+
+ @Test
+ public void testInvalidSslConfig() throws
MetricsProviderLifeCycleException {
+ assertThrows(MetricsProviderLifeCycleException.class, () -> {
+ PrometheusMetricsProvider provider = new
PrometheusMetricsProvider();
+ Properties configuration = new Properties();
+ String testDataPath = System.getProperty("test.data.dir",
"src/test/resources/data");
+ configuration.setProperty("httpsPort", "50514");
+ //keystore missing
+ configuration.setProperty("ssl.keyStore.password", "testpass");
+ configuration.setProperty("ssl.trustStore.location", testDataPath
+ "/ssl/server_truststore.jks");
+ configuration.setProperty("ssl.trustStore.password", "testpass");
+ provider.configure(configuration);
+ provider.start();
+ });
+ }
}
diff --git
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/README.md
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/README.md
new file mode 100644
index 000000000..4e1aa7484
--- /dev/null
+++
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/README.md
@@ -0,0 +1,5 @@
+SSL test data
+===================
+
+Testing client/server keystore, password is "testpass".
+Testing client/server truststore, password is "testpass".
diff --git
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/client_keystore.jks
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/client_keystore.jks
new file mode 100644
index 000000000..8636f41dd
Binary files /dev/null and
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/client_keystore.jks
differ
diff --git
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/client_truststore.jks
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/client_truststore.jks
new file mode 100644
index 000000000..3e5893d0b
Binary files /dev/null and
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/client_truststore.jks
differ
diff --git
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/server_keystore.jks
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/server_keystore.jks
new file mode 100644
index 000000000..d524c6268
Binary files /dev/null and
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/server_keystore.jks
differ
diff --git
a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/server_truststore.jks
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/server_truststore.jks
new file mode 100644
index 000000000..faa322836
Binary files /dev/null and
b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/server_truststore.jks
differ
