This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new c1c9c268f3 JAMES-2287 HashBlobId.Factory support encoding type (#1632)
c1c9c268f3 is described below
commit c1c9c268f3f42a18387b3a2bba2e08a3270eb956
Author: vttran <[email protected]>
AuthorDate: Mon Jul 10 09:01:01 2023 +0700
JAMES-2287 HashBlobId.Factory support encoding type (#1632)
---
.../docs/modules/ROOT/pages/configure/jvm.adoc | 12 ++
.../java/org/apache/james/blob/api/HashBlobId.java | 33 +++++-
.../org/apache/james/blob/api/HashBlobIdTest.java | 42 ++++++-
server/blob/blob-s3/pom.xml | 5 +
.../james/blob/objectstorage/aws/S3MinioTest.java | 123 +++++++++++++++++++++
src/site/xdoc/server/config-system.xml | 3 +
6 files changed, 214 insertions(+), 4 deletions(-)
diff --git
a/server/apps/distributed-app/docs/modules/ROOT/pages/configure/jvm.adoc
b/server/apps/distributed-app/docs/modules/ROOT/pages/configure/jvm.adoc
index f0440ab4b2..d95acb2414 100644
--- a/server/apps/distributed-app/docs/modules/ROOT/pages/configure/jvm.adoc
+++ b/server/apps/distributed-app/docs/modules/ROOT/pages/configure/jvm.adoc
@@ -53,3 +53,15 @@ james.protocols.mdc.hostname=false
Optional. Boolean. Defaults to true.
+== Change the encoding type used for the blobId
+
+By default, the blobId is encoded in base64 url. The property
`james.blob.id.hash.encoding` allows to change the encoding type.
+The support value are: base16, hex, base32, base32Hex, base64, base64Url.
+
+Ex in `jvm.properties`
+----
+james.blob.id.hash.encoding=base16
+----
+
+Optional. String. Defaults to base64Url.
+
diff --git
a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java
b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java
index e716df4040..55c1329e27 100644
---
a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java
+++
b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java
@@ -20,7 +20,7 @@
package org.apache.james.blob.api;
import java.io.IOException;
-import java.util.Base64;
+import java.util.Optional;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
@@ -29,11 +29,22 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
+import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteSource;
public class HashBlobId implements BlobId {
+ private static final String HASH_BLOB_ID_ENCODING_TYPE_PROPERTY =
"james.blob.id.hash.encoding";
+ private static final BaseEncoding HASH_BLOB_ID_ENCODING_DEFAULT =
BaseEncoding.base64Url();
public static class Factory implements BlobId.Factory {
+ private final BaseEncoding baseEncoding;
+
+ public Factory() {
+ this.baseEncoding =
Optional.ofNullable(System.getProperty(HASH_BLOB_ID_ENCODING_TYPE_PROPERTY))
+ .map(Factory::baseEncodingFrom)
+ .orElse(HASH_BLOB_ID_ENCODING_DEFAULT);
+ }
+
@Override
public HashBlobId forPayload(byte[] payload) {
Preconditions.checkArgument(payload != null);
@@ -51,7 +62,7 @@ public class HashBlobId implements BlobId {
private HashBlobId base64(HashCode hashCode) {
byte[] bytes = hashCode.asBytes();
- return new HashBlobId(Base64.getEncoder().encodeToString(bytes));
+ return new HashBlobId(baseEncoding.encode(bytes));
}
@Override
@@ -59,6 +70,24 @@ public class HashBlobId implements BlobId {
Preconditions.checkArgument(!Strings.isNullOrEmpty(id));
return new HashBlobId(id);
}
+
+ private static BaseEncoding baseEncodingFrom(String encodingType) {
+ switch (encodingType) {
+ case "base16":
+ case "hex":
+ return BaseEncoding.base16();
+ case "base64":
+ return BaseEncoding.base64();
+ case "base64Url":
+ return BaseEncoding.base64Url();
+ case "base32":
+ return BaseEncoding.base32();
+ case "base32Hex":
+ return BaseEncoding.base32Hex();
+ default:
+ throw new IllegalArgumentException("Unknown encoding type:
" + encodingType);
+ }
+ }
}
private final String id;
diff --git
a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java
b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java
index 026a57d9f8..88fe109973 100644
---
a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java
+++
b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java
@@ -23,16 +23,26 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.nio.charset.StandardCharsets;
+import java.util.stream.Stream;
import org.apache.james.util.ClassLoaderUtils;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
class HashBlobIdTest {
private static final HashBlobId.Factory BLOB_ID_FACTORY = new
HashBlobId.Factory();
+ @BeforeEach
+ void beforeEach(){
+ System.clearProperty("james.blob.id.hash.encoding");
+ }
+
@Test
void shouldRespectBeanContract() {
EqualsVerifier.forClass(HashBlobId.class).verify();
@@ -67,14 +77,42 @@ class HashBlobIdTest {
void forPayloadShouldHashEmptyArray() {
BlobId blobId = BLOB_ID_FACTORY.forPayload(new byte[0]);
-
assertThat(blobId.asString()).isEqualTo("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=");
+
assertThat(blobId.asString()).isEqualTo("47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU=");
}
@Test
void forPayloadShouldHashArray() {
BlobId blobId =
BLOB_ID_FACTORY.forPayload("content".getBytes(StandardCharsets.UTF_8));
-
assertThat(blobId.asString()).isEqualTo("7XACtDnprIRfIjV9giusFERzD722AW0+yUMil7nsn3M=");
+
assertThat(blobId.asString()).isEqualTo("7XACtDnprIRfIjV9giusFERzD722AW0-yUMil7nsn3M=");
+ }
+
+
+ @ParameterizedTest
+ @MethodSource("encodingTypeAndExpectedHash")
+ void forPayloadShouldSupportEncodingWhenConfigured(String encoding, String
expectedHash) {
+ System.setProperty("james.blob.id.hash.encoding", encoding);
+ BlobId blobId = new
HashBlobId.Factory().forPayload("content".getBytes(StandardCharsets.UTF_8));
+ assertThat(blobId.asString()).isEqualTo(expectedHash);
+ }
+
+ static Stream<Arguments> encodingTypeAndExpectedHash() {
+ return Stream.of(
+ Arguments.of("base16",
"ED7002B439E9AC845F22357D822BAC1444730FBDB6016D3EC9432297B9EC9F73"),
+ Arguments.of("hex",
"ED7002B439E9AC845F22357D822BAC1444730FBDB6016D3EC9432297B9EC9F73"),
+ Arguments.of("base32",
"5VYAFNBZ5GWIIXZCGV6YEK5MCRCHGD55WYAW2PWJIMRJPOPMT5ZQ===="),
+ Arguments.of("base64",
"7XACtDnprIRfIjV9giusFERzD722AW0+yUMil7nsn3M="),
+ Arguments.of("base64Url",
"7XACtDnprIRfIjV9giusFERzD722AW0-yUMil7nsn3M="),
+ Arguments.of("base32",
"5VYAFNBZ5GWIIXZCGV6YEK5MCRCHGD55WYAW2PWJIMRJPOPMT5ZQ===="),
+ Arguments.of("base32Hex",
"TLO05D1PT6M88NP26LUO4ATC2H2763TTMO0MQFM98CH9FEFCJTPG===="));
+ }
+
+ @Test
+ void newFactoryShouldFailWhenInvalidEncoding() {
+ System.setProperty("james.blob.id.hash.encoding", "invalid");
+ assertThatThrownBy(HashBlobId.Factory::new)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Unknown encoding type: invalid");
}
@Test
diff --git a/server/blob/blob-s3/pom.xml b/server/blob/blob-s3/pom.xml
index e56fc0da9d..951301625f 100644
--- a/server/blob/blob-s3/pom.xml
+++ b/server/blob/blob-s3/pom.xml
@@ -105,6 +105,11 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
diff --git
a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java
b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java
new file mode 100644
index 0000000000..3c3ff47c97
--- /dev/null
+++
b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java
@@ -0,0 +1,123 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws;
+
+import static org.apache.james.blob.api.BlobStoreDAOFixture.SHORT_BYTEARRAY;
+import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BLOB_ID;
+import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BUCKET_NAME;
+import static
org.apache.james.blob.objectstorage.aws.DockerAwsS3Container.ACCESS_KEY_ID;
+import static
org.apache.james.blob.objectstorage.aws.DockerAwsS3Container.SECRET_ACCESS_KEY;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.BlobStoreDAO;
+import org.apache.james.blob.api.BlobStoreDAOContract;
+import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.blob.api.TestBlobId;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+@Testcontainers
+public class S3MinioTest implements BlobStoreDAOContract {
+
+ private static final int MINIO_PORT = 9000;
+ private static S3BlobStoreDAO testee;
+
+ @Container
+ private static final GenericContainer<?> minioContainer = new
GenericContainer<>("quay.io/minio/minio")
+ .withExposedPorts(MINIO_PORT)
+ .withEnv("MINIO_ROOT_USER", ACCESS_KEY_ID)
+ .withEnv("MINIO_ROOT_PASSWORD", SECRET_ACCESS_KEY)
+ .withCommand("server", "/data", "--console-address", ":9090")
+ .withCreateContainerCmdModifier(createContainerCmd ->
createContainerCmd.withName("james-minio-s3-test"));
+
+
+ @BeforeAll
+ static void setUp() {
+ AwsS3AuthConfiguration authConfiguration =
AwsS3AuthConfiguration.builder()
+ .endpoint(URI.create(String.format("http://%s:%s/";,
minioContainer.getHost(), minioContainer.getMappedPort(MINIO_PORT))))
+ .accessKeyId(ACCESS_KEY_ID)
+ .secretKey(SECRET_ACCESS_KEY)
+ .build();
+
+ S3BlobStoreConfiguration s3Configuration =
S3BlobStoreConfiguration.builder()
+ .authConfiguration(authConfiguration)
+ .region(DockerAwsS3Container.REGION)
+ .build();
+
+ testee = new S3BlobStoreDAO(s3Configuration, new TestBlobId.Factory());
+ }
+
+ @AfterAll
+ static void tearDownClass() {
+ testee.close();
+ }
+
+ @AfterEach
+ void tearDown() {
+ testee.deleteAllBuckets().block();
+ }
+
+ @Override
+ public BlobStoreDAO testee() {
+ return testee;
+ }
+
+ @Test
+ void saveWillThrowWhenBlobIdHasSlashCharacters() {
+ BlobId invalidBlobId = new TestBlobId("test-blob//id");
+ assertThatThrownBy(() -> Mono.from(testee.save(TEST_BUCKET_NAME,
invalidBlobId, SHORT_BYTEARRAY)).block())
+ .isInstanceOf(S3Exception.class)
+ .hasMessageContaining("Object name contains unsupported
characters");
+ }
+
+ @Test
+ void saveShouldWorkWhenValidBlobId() {
+ Mono.from(testee.save(TEST_BUCKET_NAME, TEST_BLOB_ID,
SHORT_BYTEARRAY)).block();
+ assertThat(Mono.from(testee.readBytes(TEST_BUCKET_NAME,
TEST_BLOB_ID)).block()).isEqualTo(SHORT_BYTEARRAY);
+ }
+
+ @Test
+ void saveShouldWorkForAllPayloadsWithHash() {
+ Flux.range(0, 1000)
+ .concatMap(i -> {
+ byte[] payload =
RandomStringUtils.random(128).getBytes(StandardCharsets.UTF_8);
+ BlobId blobId = new HashBlobId.Factory().forPayload(payload);
+ return testee.save(TEST_BUCKET_NAME, blobId, payload);
+ })
+ .then()
+ .block();
+ }
+
+}
diff --git a/src/site/xdoc/server/config-system.xml
b/src/site/xdoc/server/config-system.xml
index 55650a198a..86b499b03a 100644
--- a/src/site/xdoc/server/config-system.xml
+++ b/src/site/xdoc/server/config-system.xml
@@ -234,6 +234,9 @@
Should we add the host in the MDC logging context for
incoming IMAP, SMTP, POP3? Doing so, a DNS resolution
is attempted for each incoming connection, which can be
costly. Remote IP is always added to the logging context.</dd>
+ <dt><strong>james.blob.id.hash.encoding</strong></dt>
+ <dd>Optional. String. Defaults to base64Url. <br/>
+ The encoding type when encode blobId. The support value
are: base16, hex, base32, base32Hex, base64, base64Url.</dd>
</dl>
</subsection>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
