This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push: new 398aa92 JAMES-3673 : Separate trust store for S3 (#751) 398aa92 is described below commit 398aa9276d9f90eb2da06de67ab0601f895d9609 Author: Karsten Otto <40171964+ott...@users.noreply.github.com> AuthorDate: Mon Nov 22 07:38:09 2021 +0100 JAMES-3673 : Separate trust store for S3 (#751) --- .../modules/ROOT/pages/configure/blobstore.adoc | 12 +++ .../sample-configuration/blob.properties | 7 ++ .../sample-configuration/blob.properties | 7 ++ .../objectstorage/aws/AwsS3AuthConfiguration.java | 92 +++++++++++++++++++++- .../blob/objectstorage/aws/S3BlobStoreDAO.java | 33 ++++++++ .../aws/AwsS3AuthConfigurationTest.java | 31 ++++++++ .../aws/s3/AwsS3ConfigurationReader.java | 8 ++ .../aws/s3/AwsS3ConfigurationReaderTest.java | 31 ++++++++ src/site/xdoc/server/config-blobstore.xml | 12 +++ 9 files changed, 229 insertions(+), 4 deletions(-) diff --git a/server/apps/distributed-app/docs/modules/ROOT/pages/configure/blobstore.adoc b/server/apps/distributed-app/docs/modules/ROOT/pages/configure/blobstore.adoc index f63b7e5..d8e58c2 100644 --- a/server/apps/distributed-app/docs/modules/ROOT/pages/configure/blobstore.adoc +++ b/server/apps/distributed-app/docs/modules/ROOT/pages/configure/blobstore.adoc @@ -127,6 +127,18 @@ Maximum size of stored objects expressed in bytes. | objectstorage.s3.http.concurrency | Allow setting the number of concurrent HTTP requests allowed by the Netty driver. + +| objectstorage.s3.truststore.path +| optional: Verify the S3 server certificate against this trust store file. + +| objectstorage.s3.truststore.type +| optional: Specify the type of the trust store, e.g. JKS, PKCS12 + +| objectstorage.s3.truststore.secret +| optional: Use this secret/password to access the trust store; default none + +| objectstorage.s3.truststore.algorithm +| optional: Use this specific trust store algorithm; default SunX509 |=== ==== Buckets Configuration diff --git a/server/apps/distributed-app/sample-configuration/blob.properties b/server/apps/distributed-app/sample-configuration/blob.properties index 58e8df5..56a1d91 100644 --- a/server/apps/distributed-app/sample-configuration/blob.properties +++ b/server/apps/distributed-app/sample-configuration/blob.properties @@ -73,6 +73,13 @@ objectstorage.s3.accessKeyId=accessKey1 # Mandatory if you choose s3 storage service, secret key configured in S3 objectstorage.s3.secretKey=secretKey1 +# Optional if you choose s3 storage service: The trust store file, secret, and algorithm to use +# when connecting to the storage service. If not specified falls back to Java defaults. +#objectstorage.s3.truststore.path= +#objectstorage.s3.truststore.type=JKS +#objectstorage.s3.truststore.secret= +#objectstorage.s3.truststore.algorithm=SunX509 + # ============================================ Blobs Exporting ============================================== # Read https://james.apache.org/server/config-blob-export.html for further details diff --git a/server/apps/distributed-pop3-app/sample-configuration/blob.properties b/server/apps/distributed-pop3-app/sample-configuration/blob.properties index 58e8df5..56a1d91 100644 --- a/server/apps/distributed-pop3-app/sample-configuration/blob.properties +++ b/server/apps/distributed-pop3-app/sample-configuration/blob.properties @@ -73,6 +73,13 @@ objectstorage.s3.accessKeyId=accessKey1 # Mandatory if you choose s3 storage service, secret key configured in S3 objectstorage.s3.secretKey=secretKey1 +# Optional if you choose s3 storage service: The trust store file, secret, and algorithm to use +# when connecting to the storage service. If not specified falls back to Java defaults. +#objectstorage.s3.truststore.path= +#objectstorage.s3.truststore.type=JKS +#objectstorage.s3.truststore.secret= +#objectstorage.s3.truststore.algorithm=SunX509 + # ============================================ Blobs Exporting ============================================== # Read https://james.apache.org/server/config-blob-export.html for further details diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3AuthConfiguration.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3AuthConfiguration.java index 4305d1f..ecc4014 100644 --- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3AuthConfiguration.java +++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3AuthConfiguration.java @@ -20,6 +20,7 @@ package org.apache.james.blob.objectstorage.aws; import java.net.URI; +import java.util.Optional; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; @@ -53,10 +54,55 @@ public class AwsS3AuthConfiguration { private final String accessKeyId; private final String secretKey; + private Optional<String> trustStorePath; + private Optional<String> trustStoreType; + private Optional<String> trustStoreSecret; + private Optional<String> trustStoreAlgorithm; + public ReadyToBuild(URI endpoint, String accessKeyId, String secretKey) { this.endpoint = endpoint; this.accessKeyId = accessKeyId; this.secretKey = secretKey; + this.trustStorePath = Optional.empty(); + this.trustStoreType = Optional.empty(); + this.trustStoreSecret = Optional.empty(); + this.trustStoreAlgorithm = Optional.empty(); + } + + public ReadyToBuild trustStorePath(Optional<String> trustStorePath) { + this.trustStorePath = trustStorePath; + return this; + } + + public ReadyToBuild trustStorePath(String trustStorePath) { + return trustStorePath(Optional.ofNullable(trustStorePath)); + } + + public ReadyToBuild trustStoreType(Optional<String> trustStoreType) { + this.trustStoreType = trustStoreType; + return this; + } + + public ReadyToBuild trustStoreType(String trustStoreType) { + return trustStoreType(Optional.ofNullable(trustStoreType)); + } + + public ReadyToBuild trustStoreSecret(Optional<String> trustStoreSecret) { + this.trustStoreSecret = trustStoreSecret; + return this; + } + + public ReadyToBuild trustStoreSecret(String trustStoreSecret) { + return trustStoreSecret(Optional.ofNullable(trustStoreSecret)); + } + + public ReadyToBuild trustStoreAlgorithm(Optional<String> trustStoreAlgorithm) { + this.trustStoreAlgorithm = trustStoreAlgorithm; + return this; + } + + public ReadyToBuild trustStoreAlgorithm(String trustStoreAlgorithm) { + return trustStoreAlgorithm(Optional.ofNullable(trustStoreAlgorithm)); } public AwsS3AuthConfiguration build() { @@ -68,7 +114,8 @@ public class AwsS3AuthConfiguration { Preconditions.checkNotNull(secretKey, "'secretKey' is mandatory"); Preconditions.checkArgument(!secretKey.isEmpty(), "'secretKey' is mandatory"); - return new AwsS3AuthConfiguration(endpoint, accessKeyId, secretKey); + return new AwsS3AuthConfiguration(endpoint, accessKeyId, secretKey, + trustStorePath, trustStoreType, trustStoreSecret, trustStoreAlgorithm); } } } @@ -77,12 +124,25 @@ public class AwsS3AuthConfiguration { private final String accessKeyId; private final String secretKey; + private final Optional<String> trustStorePath; + private final Optional<String> trustStoreType; + private final Optional<String> trustStoreSecret; + private final Optional<String> trustStoreAlgorithm; + private AwsS3AuthConfiguration(URI endpoint, String accessKeyId, - String secretKey) { + String secretKey, + Optional<String> trustStorePath, + Optional<String> trustStoreType, + Optional<String> trustStoreSecret, + Optional<String> trustStoreAlgorithm) { this.endpoint = endpoint; this.accessKeyId = accessKeyId; this.secretKey = secretKey; + this.trustStorePath = trustStorePath; + this.trustStoreType = trustStoreType; + this.trustStoreSecret = trustStoreSecret; + this.trustStoreAlgorithm = trustStoreAlgorithm; } public URI getEndpoint() { @@ -97,20 +157,41 @@ public class AwsS3AuthConfiguration { return secretKey; } + public Optional<String> getTrustStorePath() { + return trustStorePath; + } + + public Optional<String> getTrustStoreType() { + return trustStoreType; + } + + public Optional<String> getTrustStoreSecret() { + return trustStoreSecret; + } + + public Optional<String> getTrustStoreAlgorithm() { + return trustStoreAlgorithm; + } + @Override public final boolean equals(Object o) { if (o instanceof AwsS3AuthConfiguration) { AwsS3AuthConfiguration that = (AwsS3AuthConfiguration) o; return Objects.equal(endpoint, that.endpoint) && Objects.equal(accessKeyId, that.accessKeyId) && - Objects.equal(secretKey, that.secretKey); + Objects.equal(secretKey, that.secretKey) && + Objects.equal(trustStorePath, that.trustStorePath) && + Objects.equal(trustStoreType, that.trustStoreType) && + Objects.equal(trustStoreSecret, that.trustStoreSecret) && + Objects.equal(trustStoreAlgorithm, that.trustStoreAlgorithm); } return false; } @Override public final int hashCode() { - return Objects.hashCode(endpoint, accessKeyId, secretKey); + return Objects.hashCode(endpoint, accessKeyId, secretKey, + trustStorePath, trustStoreType, trustStoreSecret, trustStoreAlgorithm); } @Override @@ -119,6 +200,9 @@ public class AwsS3AuthConfiguration { .add("endpoint", endpoint) .add("accessKeyId", accessKeyId) .add("secretKey", secretKey) + .add("trustStorePath", trustStorePath) + .add("trustStoreSecret", trustStoreSecret) + .add("trustStoreAlgorithm", trustStoreAlgorithm) .toString(); } } diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java index 98b1fb9..ce2d00d 100644 --- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java +++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java @@ -22,9 +22,12 @@ package org.apache.james.blob.objectstorage.aws; import static org.apache.james.util.ReactorUtils.DEFAULT_CONCURRENCY; import java.io.Closeable; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.KeyStore; import java.time.Duration; import java.util.Collection; import java.util.List; @@ -32,6 +35,7 @@ import java.util.concurrent.CompletableFuture; import javax.annotation.PreDestroy; import javax.inject.Inject; +import javax.net.ssl.TrustManagerFactory; import org.apache.commons.io.IOUtils; import org.apache.james.blob.api.BlobId; @@ -62,6 +66,7 @@ import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.SdkPublisher; import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.http.TlsTrustManagersProvider; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Configuration; @@ -103,6 +108,7 @@ public class S3BlobStoreDAO implements BlobStoreDAO, Startable, Closeable { .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create(authConfiguration.getAccessKeyId(), authConfiguration.getSecretKey()))) .httpClientBuilder(NettyNioAsyncHttpClient.builder() + .tlsTrustManagersProvider(getTrustManagerProvider(configuration.getSpecificAuthConfiguration())) .maxConcurrency(configuration.getHttpConcurrency()) .maxPendingConnectionAcquires(10_000)) .endpointOverride(authConfiguration.getEndpoint()) @@ -116,6 +122,33 @@ public class S3BlobStoreDAO implements BlobStoreDAO, Startable, Closeable { .build(); } + private TlsTrustManagersProvider getTrustManagerProvider(AwsS3AuthConfiguration configuration) { + try { + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance( + configuration.getTrustStoreAlgorithm().orElse(TrustManagerFactory.getDefaultAlgorithm())); + KeyStore trustStore = loadTrustStore(configuration); + trustManagerFactory.init(trustStore); + return trustManagerFactory::getTrustManagers; + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + + private KeyStore loadTrustStore(AwsS3AuthConfiguration configuration) { + if (configuration.getTrustStorePath().isEmpty()) { + return null; // use java default truststore + } + try (FileInputStream trustStoreStream = new FileInputStream(configuration.getTrustStorePath().get())) { + char[] secret = configuration.getTrustStoreSecret().map(String::toCharArray).orElse(null); + KeyStore trustStore = KeyStore.getInstance( + configuration.getTrustStoreType().orElse(KeyStore.getDefaultType())); + trustStore.load(trustStoreStream, secret); + return trustStore; + } catch (GeneralSecurityException | IOException e) { + throw new RuntimeException(e); + } + } + @Override @PreDestroy public void close() { diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3AuthConfigurationTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3AuthConfigurationTest.java index 16a5837..ce45923 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3AuthConfigurationTest.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3AuthConfigurationTest.java @@ -33,6 +33,10 @@ public class AwsS3AuthConfigurationTest { private static final URI ENDPOINT = URI.create("http://myEndpoint"); private static final String ACCESS_KEY_ID = "myAccessKeyId"; private static final String SECRET_KEY = "mySecretKey"; + private static final String TRUST_STORE_PATH = "/where/ever/truststore.p12"; + private static final String TRUST_STORE_TYPE = "PKCS12"; + private static final String TRUST_STORE_SECRET = "myTrustStoreSecret"; + private static final String TRUST_STORE_ALGORITHM = "myTrustStoreAlgorithm"; @Test public void credentialsShouldRespectBeanContract() { @@ -100,12 +104,39 @@ public class AwsS3AuthConfigurationTest { .endpoint(ENDPOINT) .accessKeyId(ACCESS_KEY_ID) .secretKey(SECRET_KEY) + .trustStorePath(TRUST_STORE_PATH) + .trustStoreType(TRUST_STORE_TYPE) + .trustStoreSecret(TRUST_STORE_SECRET) + .trustStoreAlgorithm(TRUST_STORE_ALGORITHM) .build(); assertSoftly(softly -> { softly.assertThat(configuration.getEndpoint()).isEqualTo(ENDPOINT); softly.assertThat(configuration.getAccessKeyId()).isEqualTo(ACCESS_KEY_ID); softly.assertThat(configuration.getSecretKey()).isEqualTo(SECRET_KEY); + softly.assertThat(configuration.getTrustStorePath()).hasValue(TRUST_STORE_PATH); + softly.assertThat(configuration.getTrustStoreType()).hasValue(TRUST_STORE_TYPE); + softly.assertThat(configuration.getTrustStoreSecret()).hasValue(TRUST_STORE_SECRET); + softly.assertThat(configuration.getTrustStoreAlgorithm()).hasValue(TRUST_STORE_ALGORITHM); + }); + } + + @Test + public void builderShouldWorkWithoutOptionals() { + AwsS3AuthConfiguration configuration = AwsS3AuthConfiguration.builder() + .endpoint(ENDPOINT) + .accessKeyId(ACCESS_KEY_ID) + .secretKey(SECRET_KEY) + .build(); + + assertSoftly(softly -> { + softly.assertThat(configuration.getEndpoint()).isEqualTo(ENDPOINT); + softly.assertThat(configuration.getAccessKeyId()).isEqualTo(ACCESS_KEY_ID); + softly.assertThat(configuration.getSecretKey()).isEqualTo(SECRET_KEY); + softly.assertThat(configuration.getTrustStorePath()).isNotPresent(); + softly.assertThat(configuration.getTrustStoreType()).isNotPresent(); + softly.assertThat(configuration.getTrustStoreSecret()).isNotPresent(); + softly.assertThat(configuration.getTrustStoreAlgorithm()).isNotPresent(); }); } } diff --git a/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/aws/s3/AwsS3ConfigurationReader.java b/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/aws/s3/AwsS3ConfigurationReader.java index c400e75..50ba125 100644 --- a/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/aws/s3/AwsS3ConfigurationReader.java +++ b/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/aws/s3/AwsS3ConfigurationReader.java @@ -30,6 +30,10 @@ public class AwsS3ConfigurationReader { static final String OBJECTSTORAGE_ENDPOINT = "objectstorage.s3.endPoint"; static final String OBJECTSTORAGE_ACCESKEYID = "objectstorage.s3.accessKeyId"; static final String OBJECTSTORAGE_SECRETKEY = "objectstorage.s3.secretKey"; + static final String OBJECTSTORAGE_TRUSTSTORE_PATH = "objectstorage.s3.truststore.path"; + static final String OBJECTSTORAGE_TRUSTSTORE_TYPE = "objectstorage.s3.truststore.type"; + static final String OBJECTSTORAGE_TRUSTSTORE_SECRET = "objectstorage.s3.truststore.secret"; + static final String OBJECTSTORAGE_TRUSTSTORE_ALGORITHM = "objectstorage.s3.truststore.algorithm"; public static AwsS3AuthConfiguration from(Configuration configuration) { String endpoint = configuration.getString(OBJECTSTORAGE_ENDPOINT); @@ -41,6 +45,10 @@ public class AwsS3ConfigurationReader { .endpoint(URI.create(endpoint)) .accessKeyId(configuration.getString(OBJECTSTORAGE_ACCESKEYID)) .secretKey(configuration.getString(OBJECTSTORAGE_SECRETKEY)) + .trustStorePath(configuration.getString(OBJECTSTORAGE_TRUSTSTORE_PATH)) + .trustStoreType(configuration.getString(OBJECTSTORAGE_TRUSTSTORE_TYPE)) + .trustStoreSecret(configuration.getString(OBJECTSTORAGE_TRUSTSTORE_SECRET)) + .trustStoreAlgorithm(configuration.getString(OBJECTSTORAGE_TRUSTSTORE_ALGORITHM)) .build(); } } diff --git a/server/container/guice/blob/s3/src/test/java/org/apache/james/modules/objectstorage/aws/s3/AwsS3ConfigurationReaderTest.java b/server/container/guice/blob/s3/src/test/java/org/apache/james/modules/objectstorage/aws/s3/AwsS3ConfigurationReaderTest.java index 7bf39db..21edf20 100644 --- a/server/container/guice/blob/s3/src/test/java/org/apache/james/modules/objectstorage/aws/s3/AwsS3ConfigurationReaderTest.java +++ b/server/container/guice/blob/s3/src/test/java/org/apache/james/modules/objectstorage/aws/s3/AwsS3ConfigurationReaderTest.java @@ -67,6 +67,37 @@ class AwsS3ConfigurationReaderTest { configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_ACCESKEYID, accessKeyId); String secretKey = "mySecretKey"; configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_SECRETKEY, secretKey); + String trustStorePath = "/some/where/truststore.p12"; + configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_TRUSTSTORE_PATH, trustStorePath); + String trustStoreType = "PKCS12"; + configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_TRUSTSTORE_TYPE, trustStoreType); + String trustStoreSecret = "myTrustStoreSecret"; + configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_TRUSTSTORE_SECRET, trustStoreSecret); + String trustStoreAlgorithm = "myTrustStoreAlgorithm"; + configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_TRUSTSTORE_ALGORITHM, trustStoreAlgorithm); + + AwsS3AuthConfiguration expected = AwsS3AuthConfiguration.builder() + .endpoint(endpoint) + .accessKeyId(accessKeyId) + .secretKey(secretKey) + .trustStorePath(trustStorePath) + .trustStoreType(trustStoreType) + .trustStoreSecret(trustStoreSecret) + .trustStoreAlgorithm(trustStoreAlgorithm) + .build(); + AwsS3AuthConfiguration authConfiguration = AwsS3ConfigurationReader.from(configuration); + assertThat(authConfiguration).isEqualTo(expected); + } + + @Test + void fromShouldWorkWithoutOptionals() { + Configuration configuration = new PropertiesConfiguration(); + URI endpoint = URI.create("http://myEndpoint"); + configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_ENDPOINT, endpoint); + String accessKeyId = "myAccessKeyId"; + configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_ACCESKEYID, accessKeyId); + String secretKey = "mySecretKey"; + configuration.addProperty(AwsS3ConfigurationReader.OBJECTSTORAGE_SECRETKEY, secretKey); AwsS3AuthConfiguration expected = AwsS3AuthConfiguration.builder() .endpoint(endpoint) diff --git a/src/site/xdoc/server/config-blobstore.xml b/src/site/xdoc/server/config-blobstore.xml index 97e830e..c4f12c1 100644 --- a/src/site/xdoc/server/config-blobstore.xml +++ b/src/site/xdoc/server/config-blobstore.xml @@ -167,6 +167,18 @@ generate salt with : openssl rand -hex 16 <dt><strong>objectstorage.s3.http.concurrency</strong></dt> <dd>Allow setting the number of concurrent HTTP requests allowed by the Netty driver.</dd> + + <dt><strong>objectstorage.s3.truststore.path</strong></dt> + <dd><i>optional:</i> Verify the S3 server certificate against this trust store file.</dd> + + <dt><strong>objectstorage.s3.truststore.type</strong></dt> + <dd><i>optional:</i> Specify the type of the trust store, e.g. JKS, PKCS12</dd> + + <dt><strong>objectstorage.s3.truststore.secret</strong></dt> + <dd><i>optional:</i> Use this secret/password to access the trust store; default none</dd> + + <dt><strong>objectstorage.s3.truststore.algorithm</strong></dt> + <dd><i>optional:</i> Use this specific trust store algorithm; default SunX509</dd> </dl> </subsection> </subsection> --------------------------------------------------------------------- To unsubscribe, e-mail: notifications-unsubscr...@james.apache.org For additional commands, e-mail: notifications-h...@james.apache.org