This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 440008970 [CELEBORN-1346] Add build changes and test resources for ssl
support
440008970 is described below
commit 44000897087bda96aafdeb29676a5ab4d12d1623
Author: Mridul Muralidharan <[email protected]>
AuthorDate: Tue Mar 26 21:50:54 2024 +0800
[CELEBORN-1346] Add build changes and test resources for ssl support
### What changes were proposed in this pull request?
Build changes and test resources for enabling SSL support.
Please see #2416 for the consolidate PR with all the changes for reference.
Note: I closed the older PR #2413 and reopened this one give the repo
changes.
### Why are the changes needed?
Build dependency updates and addition of test resources for use with tests.
The specific tests leveraging these will be added in subsequent jiras
linked off of CELEBORN-1343
Splitting it up into multiple PR's to reduce the review load.
### Does this PR introduce _any_ user-facing change?
io.netty:netty-tcnative-boringssl-static is an additional dependency.
org.bouncycastle:* are test dependencies which should have no user facing
changes.
### How was this patch tested?
The overall PR #2411 passes all tests, this is specifically pulling out the
dependency changes and resources.
Closes #2417 from mridulm/build-and-test-for-tls.
Lead-authored-by: Mridul Muralidharan <[email protected]>
Co-authored-by: Mridul Muralidharan <mridulatgmail.com>
Signed-off-by: Cheng Pan <[email protected]>
---
.gitattributes | 3 +
common/pom.xml | 11 ++++
common/src/test/resources/ssl/generate_certs.sh | 63 +++++++++++++++++++++
common/src/test/resources/ssl/server.jks | Bin 0 -> 2559 bytes
common/src/test/resources/ssl/server_another.jks | Bin 0 -> 2559 bytes
.../test/resources/ssl/truststore-without-ca.jks | Bin 0 -> 32 bytes
common/src/test/resources/ssl/truststore.jks | Bin 0 -> 1975 bytes
pom.xml | 14 +++++
project/CelebornBuild.scala | 12 +++-
9 files changed, 102 insertions(+), 1 deletion(-)
diff --git a/.gitattributes b/.gitattributes
index e87caf595..23353cbc5 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -31,3 +31,6 @@ assets/img/* export-ignore
*.scala text eol=lf
*.xml text eol=lf
*.py text eol=lf
+common/src/test/resources/ssl/generate_certs.sh text
+common/src/test/resources/ssl/* -text
+worker/src/test/resources/ssl/* -text
diff --git a/common/pom.xml b/common/pom.xml
index e9e8a19f0..b7ae4cfc1 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -147,6 +147,17 @@
<artifactId>log4j-1.2-api</artifactId>
<scope>test</scope>
</dependency>
+ <!-- for SSL support -->
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk18on</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk18on</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/common/src/test/resources/ssl/generate_certs.sh
b/common/src/test/resources/ssl/generate_certs.sh
new file mode 100755
index 000000000..d83cc7bc9
--- /dev/null
+++ b/common/src/test/resources/ssl/generate_certs.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# A simple utility to (re-)generate the files within resources/ssl
+# These generated certificates are used for a variety of test scenarios for
SSL.
+# The utility ends up generating two certificates - which are saved into two
different keystores
+# The certificates generated are signed by two different CA cert's (also
generated here).
+# There are two truststores generated - the first truststore has both CA certs
as part of it
+# Hence this trust can be used to validate both client certificates.
+# The second trust store has NO CA certs in it - and so when used will fail
both the certificates.
+# Requires: "openssl" (typically the openssl package) and java "keytool" in
the PATH
+
+function gen_certs() {
+
+ openssl genrsa -out ca.key 2048
+ openssl req -x509 -new -days 9000 -key ca.key -out ca.crt -subj
"/C=US/ST=YourState/L=YourCity/O=YourOrganization/CN=MyCACert"
+ openssl genrsa -out server.key 2048
+ openssl req -new -key server.key -out server.csr -subj
"/C=US/ST=YourState/L=YourCity/O=YourOrganization/CN=MyServer"
+ openssl x509 -req -CA ca.crt -CAkey ca.key -CAcreateserial -in server.csr
-out server.crt -days 8000
+ openssl pkcs12 -export -in server.crt -inkey server.key -out keystore.p12
-name servercert -password pass:password
+ keytool -importkeystore -destkeystore server.jks -srckeystore keystore.p12
-srcstoretype PKCS12 -deststoretype pkcs12 -srcstorepass password
-deststorepass password -noprompt
+
+ keytool -import -trustcacerts -alias CACert -file ca.crt -keystore
truststore.jks -storepass password -noprompt
+
+ rm ca.srl keystore.p12 server.csr ca.key server.key server.crt
+}
+
+
+mkdir for_default
+cd for_default
+gen_certs
+cd ..
+mkdir for_secondary
+cd for_secondary
+gen_certs
+cd ..
+
+
+cp ./for_default/truststore.jks ./for_default/server.jks .
+cp ./for_secondary/server.jks ./server_another.jks
+
+
+keytool -import -trustcacerts -alias 'CACertAnother' -file
for_secondary/ca.crt -keystore ./truststore.jks -storepass password -noprompt
+
+# Copy the secondary trust store and remove the ca to generate
truststore-without-ca.jks
+cp ./for_secondary/truststore.jks ./truststore-without-ca.jks
+keytool -delete -alias 'CACert' -keystore ./truststore-without-ca.jks
-storepass password -noprompt
+
+rm -rf for_default for_secondary
diff --git a/common/src/test/resources/ssl/server.jks
b/common/src/test/resources/ssl/server.jks
new file mode 100644
index 000000000..164c9387a
Binary files /dev/null and b/common/src/test/resources/ssl/server.jks differ
diff --git a/common/src/test/resources/ssl/server_another.jks
b/common/src/test/resources/ssl/server_another.jks
new file mode 100644
index 000000000..68b1cee5f
Binary files /dev/null and b/common/src/test/resources/ssl/server_another.jks
differ
diff --git a/common/src/test/resources/ssl/truststore-without-ca.jks
b/common/src/test/resources/ssl/truststore-without-ca.jks
new file mode 100644
index 000000000..65d4b6528
Binary files /dev/null and
b/common/src/test/resources/ssl/truststore-without-ca.jks differ
diff --git a/common/src/test/resources/ssl/truststore.jks
b/common/src/test/resources/ssl/truststore.jks
new file mode 100644
index 000000000..34eebe109
Binary files /dev/null and b/common/src/test/resources/ssl/truststore.jks differ
diff --git a/pom.xml b/pom.xml
index 5876863d5..14ec1775e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@
<mockito.version>4.11.0</mockito.version>
<mockito-scalatest.version>1.17.14</mockito-scalatest.version>
<netty.version>4.1.107.Final</netty.version>
+ <bouncycastle.version>1.77</bouncycastle.version>
<protobuf.version>3.21.7</protobuf.version>
<ratis.version>2.5.1</ratis.version>
<scalatest.version>3.2.16</scalatest.version>
@@ -522,6 +523,19 @@
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>
+ <!-- for SSL support -->
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk18on</artifactId>
+ <version>${bouncycastle.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk18on</artifactId>
+ <version>${bouncycastle.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index fc7052a36..12ff97acc 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -68,6 +68,9 @@ object Dependencies {
val hikaricpVersion = "4.0.3"
val h2Version = "2.2.224"
+ // For SSL support
+ val bouncycastleVersion = "1.77"
+
// Versions for proto
val protocVersion = "3.21.7"
val protoVersion = "3.21.7"
@@ -140,6 +143,10 @@ object Dependencies {
val scalatestMockito = "org.mockito" %% "mockito-scala-scalatest" %
scalatestMockitoVersion
val scalatest = "org.scalatest" %% "scalatest" % scalatestVersion
val h2 = "com.h2database" % "h2" % h2Version
+
+ // SSL support
+ val bouncycastleBcprovJdk18on = "org.bouncycastle" % "bcprov-jdk18on" %
bouncycastleVersion % "test"
+ val bouncycastleBcpkixJdk18on = "org.bouncycastle" % "bcpkix-jdk18on" %
bouncycastleVersion % "test"
}
object CelebornCommonSettings {
@@ -394,7 +401,10 @@ object CelebornCommon {
Dependencies.jacksonDatabind,
Dependencies.jacksonAnnotations,
Dependencies.log4jSlf4jImpl % "test",
- Dependencies.log4j12Api % "test"
+ Dependencies.log4j12Api % "test",
+ // SSL support
+ Dependencies.bouncycastleBcprovJdk18on,
+ Dependencies.bouncycastleBcpkixJdk18on
) ++ commonUnitTestDependencies,
Compile / sourceGenerators += Def.task {
