This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 11c7d1080 [CELEBORN-1354] auto ssl for rpc_app transport module
11c7d1080 is described below
commit 11c7d1080c76b86061b46ae3f99ceade37086f82
Author: Mridul Muralidharan <[email protected]>
AuthorDate: Sat Apr 20 23:17:10 2024 +0800
[CELEBORN-1354] auto ssl for rpc_app transport module
### What changes were proposed in this pull request?
Support over the wire encryption between client side components for
`rpc_app` transport module without requiring keystore/truststore to be
configured.
### Why are the changes needed?
As detailed in CELEBORN-1354, support over the wire encryption between
client side components (`rpc_app` transport module) without needing each
application to be configured with certificates.
When enabled, `SSLFactory` will create a self-signed certificate and
leverage it to secure the rpc comms.
### Does this PR introduce _any_ user-facing change?
Introduces a configuration to enable/disable this behavior.
### How was this patch tested?
New tests added to validate functionality.
Closes #2469 from mridulm/support-app-auto-ssl.
Lead-authored-by: Mridul Muralidharan <[email protected]>
Co-authored-by: Mridul Muralidharan <mridulatgmail.com>
Signed-off-by: zky.zhoukeyong <[email protected]>
---
.../celeborn/common/network/TransportContext.java | 6 +-
.../celeborn/common/network/ssl/SSLFactory.java | 46 +++++++++++
.../celeborn/common/network/ssl/SSLUtils.java | 91 ++++++++++++++++++++++
.../common/network/util/TransportConf.java | 31 ++++++++
.../org/apache/celeborn/common/CelebornConf.scala | 16 ++++
.../common/network/AutoSSLRpcIntegrationSuite.java | 52 +++++++++++++
.../common/network/RpcIntegrationSuiteJ.java | 2 +-
.../celeborn/common/network/ssl/SSLUtilsSuite.java | 63 +++++++++++++++
.../common/network/ssl/SslConnectivitySuiteJ.java | 83 +++++++++++++++++++-
.../common/network/ssl/SslSampleConfigs.java | 8 ++
.../common/network/util/TransportConfSuiteJ.java | 38 +++++++++
11 files changed, 429 insertions(+), 7 deletions(-)
diff --git
a/common/src/main/java/org/apache/celeborn/common/network/TransportContext.java
b/common/src/main/java/org/apache/celeborn/common/network/TransportContext.java
index 58366a087..aa9543478 100644
---
a/common/src/main/java/org/apache/celeborn/common/network/TransportContext.java
+++
b/common/src/main/java/org/apache/celeborn/common/network/TransportContext.java
@@ -209,10 +209,12 @@ public class TransportContext implements Closeable {
private SSLFactory createSslFactory() {
if (conf.sslEnabled()) {
+
if (conf.sslEnabledAndKeysAreValid()) {
return new SSLFactory.Builder()
.requestedProtocol(conf.sslProtocol())
.requestedCiphers(conf.sslRequestedCiphers())
+ .autoSslEnabled(conf.autoSslEnabled())
.keyStore(conf.sslKeyStore(), conf.sslKeyStorePassword())
.trustStore(
conf.sslTrustStore(),
@@ -222,14 +224,14 @@ public class TransportContext implements Closeable {
.build();
} else {
logger.error(
- "SSL encryption enabled but keys not found for "
+ "SSL encryption enabled but keyStore is not configured for "
+ conf.getModuleName()
+ "! Please ensure the configured keys are present.");
throw new IllegalArgumentException(
conf.getModuleName()
+ " SSL encryption enabled for "
+ conf.getModuleName()
- + " but keys not found!");
+ + " but keyStore not configured !");
}
} else {
return null;
diff --git
a/common/src/main/java/org/apache/celeborn/common/network/ssl/SSLFactory.java
b/common/src/main/java/org/apache/celeborn/common/network/ssl/SSLFactory.java
index 443db6a9c..9612a7749 100644
---
a/common/src/main/java/org/apache/celeborn/common/network/ssl/SSLFactory.java
+++
b/common/src/main/java/org/apache/celeborn/common/network/ssl/SSLFactory.java
@@ -76,6 +76,17 @@ public class SSLFactory {
}
private void initJdkSslContext(final Builder b) throws IOException,
GeneralSecurityException {
+ // Validate invariants
+ if (b.autoSslEnabled) {
+ // this has been already validated - adding precondition check here
since we are going to
+ // overwrite these configs
+ if (null != b.keyStore || null != b.trustStore) {
+ throw new IllegalArgumentException(
+ "keystore and truststore cant be configured for auto ssl");
+ }
+ configureAutoSsl(b);
+ }
+
this.keyManagers =
null != b.keyStore ? keyManagers(b.keyStore, b.keyPassword,
b.keyStorePassword) : null;
this.trustManagers =
@@ -85,6 +96,28 @@ public class SSLFactory {
this.jdkSslContext = createSSLContext(requestedProtocol, keyManagers,
trustManagers);
}
+ /*
+ * As b.trustStore is null, credulousTrustStoreManagers will be used - and
so all
+ * certs will be accepted - and hence self-signed cert from lifecycle
manager will
+ * be accepted at executors: nothing else needs to be done to enable its
+ * use - other than generation of the certificate itself, which is what gets
done here.
+ */
+ private void configureAutoSsl(Builder b) {
+ // The keystore creation ideally be done only at Lifecycle manager, and
not in executors
+ // It does not cause issues to run it at executors, though it is not
necessary.
+ // How can be identify this scenario and ensure this is not invoked at
executors ?
+ try {
+ SSLUtils.SelfSignedCertificateConfig config = new
SSLUtils.SelfSignedCertificateConfig();
+ SSLUtils.generateSelfSignedCertificate(config);
+ // Now that we have create the self signed cert, update the builder
config
+ b.keyStore(config.keystoreFile,
config.keystorePassword).keyPassword(config.keyPassword);
+ } catch (CertificateException | KeyStoreException | IOException |
NoSuchAlgorithmException e) {
+ // Unexpected
+ throw new IllegalStateException(
+ "Unable to create self signed certificate for configuring auto ssl",
e);
+ }
+ }
+
public boolean hasKeyManagers() {
return null != keyManagers;
}
@@ -120,6 +153,7 @@ public class SSLFactory {
private String trustStorePassword;
private boolean trustStoreReloadingEnabled;
private int trustStoreReloadIntervalMs;
+ private boolean autoSslEnabled;
/**
* Sets the requested protocol, i.e., "TLSv1.2", "TLSv1.1", etc
@@ -189,6 +223,18 @@ public class SSLFactory {
return this;
}
+ /**
+ * Enables auto ssl, which is applicable only to rpc_app transport module.
When enabled,
+ * keystore and trust store should not be configured. A self signed
certificate is generated,
+ * which is used for securing the communication - and on ssl client
connection side, all
+ * certificates will be accepted. This allows for configuring network
encryption without needing
+ * to provision certificates for each application.
+ */
+ public Builder autoSslEnabled(boolean autoSslEnabled) {
+ this.autoSslEnabled = autoSslEnabled;
+ return this;
+ }
+
/**
* Builds our {@link SSLFactory}
*
diff --git
a/common/src/main/java/org/apache/celeborn/common/network/ssl/SSLUtils.java
b/common/src/main/java/org/apache/celeborn/common/network/ssl/SSLUtils.java
new file mode 100644
index 000000000..53675b73b
--- /dev/null
+++ b/common/src/main/java/org/apache/celeborn/common/network/ssl/SSLUtils.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.network.ssl;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.security.*;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.UUID;
+
+import io.netty.handler.ssl.util.SelfSignedCertificate;
+
+/** SSL util functions for use with SSLFactory */
+class SSLUtils {
+
+ static final String KEY_PAIR_ALGORITHM =
+ System.getProperty("celeborn.SSLUtils.KEY_PAIR_ALGORITHM", "RSA");
+ static final int KEY_PAIR_STRENGTH =
+ Integer.getInteger("celeborn.SSLUtils.KEY_PAIR_STRENGTH", 2048);
+ static final long CERTIFICATE_VALIDITY =
+ // default, 1 year validity
+ Long.getLong("celeborn.SSLUtils.CERTIFICATE_VALIDITY_DAYS", 365) * 24 *
60 * 60 * 1000;
+
+ static class SelfSignedCertificateConfig {
+ final File keystoreFile;
+ final String alias;
+ final String keystorePassword;
+ final String keyPassword;
+
+ SelfSignedCertificateConfig() throws IOException {
+ this(
+ createTempJksFile(),
+ UUID.randomUUID().toString(),
+ UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
+ }
+
+ SelfSignedCertificateConfig(
+ File keystoreFile, String alias, String keystorePassword, String
keyPassword) {
+ this.keystoreFile = keystoreFile;
+ this.alias = alias;
+ this.keystorePassword = keystorePassword;
+ this.keyPassword = keyPassword;
+ }
+
+ private static File createTempJksFile() throws IOException {
+ File keystoreFile = File.createTempFile("auto-ssl-", ".jks");
+ keystoreFile.deleteOnExit();
+ return keystoreFile;
+ }
+ }
+
+ static void generateSelfSignedCertificate(SelfSignedCertificateConfig config)
+ throws IOException, CertificateException, KeyStoreException,
NoSuchAlgorithmException {
+ SelfSignedCertificate cert = new SelfSignedCertificate();
+ PrivateKey privateKey = cert.key();
+ X509Certificate certificate = cert.cert();
+
+ // Create a KeyStore
+ KeyStore keyStore = KeyStore.getInstance("JKS");
+ keyStore.load(null, null); // Initialize empty keystore
+
+ keyStore.setKeyEntry(
+ config.alias,
+ privateKey,
+ config.keyPassword.toCharArray(),
+ new X509Certificate[] {certificate});
+
+ // Save the keystore
+ try (FileOutputStream outputStream = new
FileOutputStream(config.keystoreFile)) {
+ keyStore.store(outputStream, config.keystorePassword.toCharArray());
+ }
+ }
+}
diff --git
a/common/src/main/java/org/apache/celeborn/common/network/util/TransportConf.java
b/common/src/main/java/org/apache/celeborn/common/network/util/TransportConf.java
index a1286b285..f0203df4c 100644
---
a/common/src/main/java/org/apache/celeborn/common/network/util/TransportConf.java
+++
b/common/src/main/java/org/apache/celeborn/common/network/util/TransportConf.java
@@ -19,11 +19,17 @@ package org.apache.celeborn.common.network.util;
import java.io.File;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.celeborn.common.CelebornConf;
+import org.apache.celeborn.common.protocol.TransportModuleConstants;
/** A central location that tracks all the settings we expose to users. */
public class TransportConf {
+ private static final Logger logger =
LoggerFactory.getLogger(TransportConf.class);
+
private final CelebornConf celebornConf;
private final String module;
@@ -236,4 +242,29 @@ public class TransportConf {
// It's fine for the trust store to be missing, we would default to
trusting all.
return true;
}
+
+ public boolean autoSslEnabled() {
+ // auto_ssl_enable is supported only for RPC_APP_MODULE
+ if (!TransportModuleConstants.RPC_APP_MODULE.equals(module)) return false;
+
+ // auto ssl must be enabled, and there should be no keystore or trust
store configured
+ boolean autoSslEnabled = celebornConf.isAutoSslEnabled(module);
+
+ if (!autoSslEnabled) return false;
+
+ File keystore = celebornConf.sslKeyStore(module);
+ File truststore = celebornConf.sslTrustStore(module);
+
+ boolean shouldAutoSslEnabled = null == keystore && null == truststore;
+
+ if (!shouldAutoSslEnabled) {
+ logger.warn(
+ "Auto ssl for {} disabled as keystore = {} or truststore = {} was
configured",
+ module,
+ null != keystore ? keystore.getPath() : null,
+ null != truststore ? truststore.getPath() : null);
+ }
+
+ return shouldAutoSslEnabled;
+ }
}
diff --git
a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index c23703979..9e7d51d97 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -1312,6 +1312,10 @@ class CelebornConf(loadDefaults: Boolean) extends
Cloneable with Logging with Se
getSslConfig(MAX_SSL_ENCRYPTED_BLOCK_SIZE, module).toInt
}
+ def isAutoSslEnabled(module: String): Boolean = {
+ getSslConfig(AUTO_SSL_ENABLED, module)
+ }
+
// //////////////////////////////////////////////////////
// Authentication //
// //////////////////////////////////////////////////////
@@ -5059,6 +5063,18 @@ object CelebornConf extends Logging {
s"Invalid maxEncryptedBlockSize, must be a position number upto
${Int.MaxValue}")
.createWithDefaultString("64k")
+ val AUTO_SSL_ENABLED: ConfigEntry[Boolean] =
+ buildConf("celeborn.ssl.<module>.autoSslEnabled")
+ .categories("network", "ssl")
+ .version("0.5.0")
+ .internal
+ .doc("Enable auto ssl where lifecycle manager will generate a
self-signed certificate, " +
+ s"which will be trusted by all executors connecting to it. This is
applicable only for " +
+ s"${TransportModuleConstants.RPC_APP_MODULE} module, and ignored for
others. " +
+ "Additionally if truststore or keystore are present, this config is
ignored.")
+ .booleanConf
+ .createWithDefault(false)
+
val SECRET_REDACTION_PATTERN =
buildConf("celeborn.redaction.regex")
.categories("master", "worker")
diff --git
a/common/src/test/java/org/apache/celeborn/common/network/AutoSSLRpcIntegrationSuite.java
b/common/src/test/java/org/apache/celeborn/common/network/AutoSSLRpcIntegrationSuite.java
new file mode 100644
index 000000000..7e6af4548
--- /dev/null
+++
b/common/src/test/java/org/apache/celeborn/common/network/AutoSSLRpcIntegrationSuite.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.network;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.celeborn.common.CelebornConf;
+import org.apache.celeborn.common.network.ssl.SslSampleConfigs;
+import org.apache.celeborn.common.protocol.TransportModuleConstants;
+
+public class AutoSSLRpcIntegrationSuite extends RpcIntegrationSuiteJ {
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // change it to client module
+ RpcIntegrationSuiteJ.TEST_MODULE = TransportModuleConstants.RPC_APP_MODULE;
+ // set up SSL for TEST_MODULE
+ RpcIntegrationSuiteJ.initialize(
+ TestHelper.updateCelebornConfWithMap(
+ new CelebornConf(),
SslSampleConfigs.createAutoSslConfigForModule(TEST_MODULE)));
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ RpcIntegrationSuiteJ.tearDown();
+ }
+
+ @Test
+ public void validateSslConfig() {
+ // this is to ensure ssl config has been applied.
+ assertTrue(conf.sslEnabled());
+ assertTrue(conf.autoSslEnabled());
+ }
+}
diff --git
a/common/src/test/java/org/apache/celeborn/common/network/RpcIntegrationSuiteJ.java
b/common/src/test/java/org/apache/celeborn/common/network/RpcIntegrationSuiteJ.java
index 76cc0dc2c..bd2fbf830 100644
---
a/common/src/test/java/org/apache/celeborn/common/network/RpcIntegrationSuiteJ.java
+++
b/common/src/test/java/org/apache/celeborn/common/network/RpcIntegrationSuiteJ.java
@@ -43,7 +43,7 @@ import org.apache.celeborn.common.network.util.TransportConf;
import org.apache.celeborn.common.util.JavaUtils;
public class RpcIntegrationSuiteJ {
- static final String TEST_MODULE = "shuffle";
+ static String TEST_MODULE = "shuffle";
static TransportConf conf;
static TransportContext context;
static TransportServer server;
diff --git
a/common/src/test/java/org/apache/celeborn/common/network/ssl/SSLUtilsSuite.java
b/common/src/test/java/org/apache/celeborn/common/network/ssl/SSLUtilsSuite.java
new file mode 100644
index 000000000..b8c14afff
--- /dev/null
+++
b/common/src/test/java/org/apache/celeborn/common/network/ssl/SSLUtilsSuite.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.network.ssl;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.security.Key;
+import java.security.KeyStore;
+import java.security.cert.Certificate;
+import java.util.Enumeration;
+
+import org.junit.Test;
+
+public class SSLUtilsSuite {
+ @Test
+ public void testSelfSignedCertificateGeneration() throws Exception {
+ File keystoreFile = File.createTempFile("keystore-", ".jks");
+ keystoreFile.deleteOnExit();
+
+ String alias = "my-cert";
+ String keyPassword = "keypassword";
+ String storePassword = "storePassword";
+
+ SSLUtils.SelfSignedCertificateConfig config =
+ new SSLUtils.SelfSignedCertificateConfig(keystoreFile, alias,
storePassword, keyPassword);
+
+ SSLUtils.generateSelfSignedCertificate(config);
+
+ // Once jks has been created, validate its contents
+ try (FileInputStream fis = new FileInputStream(keystoreFile)) {
+ KeyStore keystore = KeyStore.getInstance("JKS");
+ keystore.load(fis, storePassword.toCharArray());
+ assertEquals(1, keystore.size());
+
+ Enumeration<String> aliasEnumeration = keystore.aliases();
+ assertTrue(aliasEnumeration.hasMoreElements());
+ assertEquals(alias, aliasEnumeration.nextElement());
+ assertFalse(aliasEnumeration.hasMoreElements());
+
+ Certificate certificate = keystore.getCertificate(alias);
+ assertNotNull(certificate);
+ Key key = keystore.getKey(alias, keyPassword.toCharArray());
+ assertNotNull(key);
+ }
+ }
+}
diff --git
a/common/src/test/java/org/apache/celeborn/common/network/ssl/SslConnectivitySuiteJ.java
b/common/src/test/java/org/apache/celeborn/common/network/ssl/SslConnectivitySuiteJ.java
index 5f8b1902c..88b9f293c 100644
---
a/common/src/test/java/org/apache/celeborn/common/network/ssl/SslConnectivitySuiteJ.java
+++
b/common/src/test/java/org/apache/celeborn/common/network/ssl/SslConnectivitySuiteJ.java
@@ -42,6 +42,7 @@ import
org.apache.celeborn.common.network.protocol.RequestMessage;
import org.apache.celeborn.common.network.server.BaseMessageHandler;
import org.apache.celeborn.common.network.server.TransportServer;
import org.apache.celeborn.common.network.util.TransportConf;
+import org.apache.celeborn.common.protocol.TransportModuleConstants;
import org.apache.celeborn.common.util.JavaUtils;
/**
@@ -185,6 +186,23 @@ public class SslConnectivitySuiteJ {
Function.identity());
}
+ @Test
+ public void testNormalConnectivityWithAuthWorks() throws Exception {
+ final Function<CelebornConf, CelebornConf> updateConf =
+ conf -> {
+ conf.set("celeborn.auth.enabled", "true");
+ return conf;
+ };
+
+ testSuccessfulConnectivity(
+ false,
+ // does not matter what these are set to
+ true,
+ true,
+ updateConf,
+ updateConf);
+ }
+
// Both server and client are on SSL, and should be able to successfully
communicate
// This is the SSL version of testNormalConnectivityWorks above.
@Test
@@ -296,13 +314,31 @@ public class SslConnectivitySuiteJ {
Function<CelebornConf, CelebornConf> postProcessServerConf,
Function<CelebornConf, CelebornConf> postProcessClientConf)
throws Exception {
+
+ testSuccessfulConnectivity(
+ TEST_MODULE,
+ enableSsl,
+ primaryConfigForServer,
+ primaryConfigForClient,
+ postProcessServerConf,
+ postProcessClientConf);
+ }
+
+ private void testSuccessfulConnectivity(
+ String module,
+ boolean enableSsl,
+ boolean primaryConfigForServer,
+ boolean primaryConfigForClient,
+ Function<CelebornConf, CelebornConf> postProcessServerConf,
+ Function<CelebornConf, CelebornConf> postProcessClientConf)
+ throws Exception {
try (TestTransportState state =
new TestTransportState(
DEFAULT_HANDLER,
createTransportConf(
- TEST_MODULE, enableSsl, primaryConfigForServer,
postProcessServerConf),
+ module, enableSsl, primaryConfigForServer,
postProcessServerConf),
createTransportConf(
- TEST_MODULE, enableSsl, primaryConfigForClient,
postProcessClientConf));
+ module, enableSsl, primaryConfigForClient,
postProcessClientConf));
TransportClient client = state.createClient()) {
String msg = " hi ";
@@ -321,13 +357,32 @@ public class SslConnectivitySuiteJ {
Function<CelebornConf, CelebornConf> postProcessServerConf,
Function<CelebornConf, CelebornConf> postProcessClientConf)
throws Exception {
+ testConnectivityFailure(
+ TEST_MODULE,
+ serverSsl,
+ clientSsl,
+ primaryConfigForServer,
+ primaryConfigForClient,
+ postProcessServerConf,
+ postProcessClientConf);
+ }
+
+ private void testConnectivityFailure(
+ String module,
+ boolean serverSsl,
+ boolean clientSsl,
+ boolean primaryConfigForServer,
+ boolean primaryConfigForClient,
+ Function<CelebornConf, CelebornConf> postProcessServerConf,
+ Function<CelebornConf, CelebornConf> postProcessClientConf)
+ throws Exception {
try (TestTransportState state =
new TestTransportState(
DEFAULT_HANDLER,
createTransportConf(
- TEST_MODULE, serverSsl, primaryConfigForServer,
postProcessServerConf),
+ module, serverSsl, primaryConfigForServer,
postProcessServerConf),
createTransportConf(
- TEST_MODULE, clientSsl, primaryConfigForClient,
postProcessClientConf));
+ module, clientSsl, primaryConfigForClient,
postProcessClientConf));
TransportClient client = state.createClient()) {
String msg = " hi ";
@@ -338,4 +393,24 @@ public class SslConnectivitySuiteJ {
// this is fine - expected to fail
}
}
+
+ @Test
+ public void testAutoSslConnectivity() throws Exception {
+ String module = TransportModuleConstants.RPC_MODULE;
+
+ final Function<CelebornConf, CelebornConf> updateConf =
+ conf -> {
+ // return a new config
+ CelebornConf celebornConf = new CelebornConf();
+ celebornConf.set("celeborn.ssl." + module + ".enabled", "true");
+ celebornConf.set("celeborn.ssl." + module + ".protocol", "TLSv1.2");
+ celebornConf.set("celeborn.ssl." + module + ".autoSslEnabled",
"true");
+ return celebornConf;
+ };
+
+ // checking nettyssl == false at both client and server does not make
sense in this context
+ // it is the same cert for both :)
+ testSuccessfulConnectivity(
+ TransportModuleConstants.RPC_APP_MODULE, true, true, true, updateConf,
updateConf);
+ }
}
diff --git
a/common/src/test/java/org/apache/celeborn/common/network/ssl/SslSampleConfigs.java
b/common/src/test/java/org/apache/celeborn/common/network/ssl/SslSampleConfigs.java
index c33baf92a..93007a936 100644
---
a/common/src/test/java/org/apache/celeborn/common/network/ssl/SslSampleConfigs.java
+++
b/common/src/test/java/org/apache/celeborn/common/network/ssl/SslSampleConfigs.java
@@ -58,6 +58,14 @@ public class SslSampleConfigs {
return createConfigMapForModule(module, false);
}
+ public static Map<String, String> createAutoSslConfigForModule(String
module) {
+ Map<String, String> confMap = new HashMap<>();
+ confMap.put("celeborn.ssl." + module + ".enabled", "true");
+ confMap.put("celeborn.ssl." + module + ".protocol", "TLSv1.2");
+ confMap.put("celeborn.ssl." + module + ".autoSslEnabled", "true");
+ return confMap;
+ }
+
private static Map<String, String> createConfigMapForModule(String module,
boolean forDefault) {
Map<String, String> confMap = new HashMap<>();
confMap.put("celeborn.ssl." + module + ".enabled", "true");
diff --git
a/common/src/test/java/org/apache/celeborn/common/network/util/TransportConfSuiteJ.java
b/common/src/test/java/org/apache/celeborn/common/network/util/TransportConfSuiteJ.java
index 92288b21a..2f4b1e0ad 100644
---
a/common/src/test/java/org/apache/celeborn/common/network/util/TransportConfSuiteJ.java
+++
b/common/src/test/java/org/apache/celeborn/common/network/util/TransportConfSuiteJ.java
@@ -28,6 +28,7 @@ import org.junit.Test;
import org.apache.celeborn.common.CelebornConf;
import org.apache.celeborn.common.network.TestHelper;
import org.apache.celeborn.common.network.ssl.SslSampleConfigs;
+import org.apache.celeborn.common.protocol.TransportModuleConstants;
public class TransportConfSuiteJ {
@@ -106,4 +107,41 @@ public class TransportConfSuiteJ {
assertEquals(module1ReloadIntervalMs,
module1TestConf.sslTrustStoreReloadIntervalMs());
assertEquals(defaultReloadIntervalMs,
module2TestConf.sslTrustStoreReloadIntervalMs());
}
+
+ @Test
+ public void testAutoSslConfiguration() {
+
+ // for rpc_app module, it can be enabled.
+ TransportConf conf =
+ new TransportConf(
+ TransportModuleConstants.RPC_APP_MODULE,
+ TestHelper.updateCelebornConfWithMap(
+ new CelebornConf(),
+ SslSampleConfigs.createAutoSslConfigForModule(
+ TransportModuleConstants.RPC_APP_MODULE)));
+ assertTrue(conf.autoSslEnabled());
+
+ // for any other module, it cant be enabled.
+ conf =
+ new TransportConf(
+ TransportModuleConstants.RPC_SERVICE_MODULE,
+ TestHelper.updateCelebornConfWithMap(
+ new CelebornConf(),
+ SslSampleConfigs.createAutoSslConfigForModule(
+ TransportModuleConstants.RPC_SERVICE_MODULE)));
+ assertFalse(conf.autoSslEnabled());
+
+ // even if set to true, it gets disabled in case
+ // keystore or truststore is specified (which the default config has
specified)
+
+ CelebornConf celebornConf = new CelebornConf();
+ TestHelper.updateCelebornConfWithMap(
+ celebornConf,
+
SslSampleConfigs.createDefaultConfigMapForModule(TransportModuleConstants.RPC_APP_MODULE));
+ celebornConf.set(
+ "celeborn.ssl." + TransportModuleConstants.RPC_APP_MODULE +
".autoSslEnabled", "true");
+
+ conf = new TransportConf(TransportModuleConstants.RPC_APP_MODULE,
celebornConf);
+ assertFalse(conf.autoSslEnabled());
+ }
}
