This is an automated email from the ASF dual-hosted git repository. quantranhong1999 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 99eb9bbd37bf882845c74da229a209e264560945 Author: Benoit TELLIER <[email protected]> AuthorDate: Sat Sep 5 00:00:22 2026 +0200 JAMES-4224 Bundle blobId naming logic in its factory --- .../mail/ContentRecoveryMessageContentSaver.java | 5 +- .../ContentRecoveryMessageContentSaverTest.java | 13 ---- .../java/org/apache/james/blob/api/BlobId.java | 12 ++++ .../org/apache/james/blob/api/BlobIdEncoding.java | 71 +++++++++++++++++++ .../org/apache/james/blob/api/PlainBlobId.java | 19 +++++ .../apache/james/blob/api/BlobIdEncodingTest.java | 82 ++++++++++++++++++++++ .../blob/api/DeduplicationBlobStoreContract.java | 23 ------ .../org/apache/james/blob/api/PlainBlobIdTest.java | 45 ++++++++++++ .../java/org/apache/james/blob/api/TestBlobId.java | 12 ++++ .../blob/deduplication/GenerationAwareBlobId.java | 11 +++ .../deduplication/MinIOGenerationAwareBlobId.java | 11 +++ .../deduplication/DeDuplicationBlobStore.scala | 44 ++---------- .../blob/deduplication/PassThroughBlobStore.scala | 7 +- .../blob/BlobMailRepositoryFactory.scala | 4 ++ 14 files changed, 278 insertions(+), 81 deletions(-) diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/ContentRecoveryMessageContentSaver.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/ContentRecoveryMessageContentSaver.java index c6a7c05443..ccf944732d 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/ContentRecoveryMessageContentSaver.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/ContentRecoveryMessageContentSaver.java @@ -22,7 +22,6 @@ package org.apache.james.mailbox.cassandra.mail; import static org.apache.james.blob.api.BlobStore.StoragePolicy.LOW_COST; import org.apache.james.blob.api.BlobId; -import org.apache.james.blob.api.BlobIdEntropy; import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.BlobStoreCacheCallback; import org.apache.james.blob.api.BlobStoreDAO; @@ -31,7 +30,6 @@ import org.apache.james.blob.api.BlobStoreDAO.BlobMetadataName; import org.apache.james.blob.api.BlobStoreDAO.BlobMetadataValue; import org.apache.james.blob.api.BlobStoreDAO.BytesBlob; -import com.google.common.io.BaseEncoding; import com.google.common.io.ByteSource; import reactor.core.publisher.Mono; @@ -56,7 +54,6 @@ import reactor.util.function.Tuples; public class ContentRecoveryMessageContentSaver implements MessageContentSaver { public static final String HEADER_BLOB_ID_SUFFIX = "_hdr"; public static final BlobMetadataName BODY_BLOB_ID = new BlobMetadataName("body-blob-id"); - private static final BaseEncoding BLOB_ID_ENCODING = BaseEncoding.base64Url().omitPadding(); private final BlobStore blobStore; private final BlobStoreDAO blobStoreDAO; @@ -93,6 +90,6 @@ public class ContentRecoveryMessageContentSaver implements MessageContentSaver { * header blob stays generation aware and is garbage collected like any other blob. */ private BlobId generateHeaderBlobId() { - return blobIdFactory.of(BLOB_ID_ENCODING.encode(BlobIdEntropy.randomBytes()) + HEADER_BLOB_ID_SUFFIX); + return blobIdFactory.random().withSuffix(HEADER_BLOB_ID_SUFFIX); } } diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/ContentRecoveryMessageContentSaverTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/ContentRecoveryMessageContentSaverTest.java index ca1e449f64..463a918c47 100644 --- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/ContentRecoveryMessageContentSaverTest.java +++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/ContentRecoveryMessageContentSaverTest.java @@ -32,7 +32,6 @@ import java.time.ZoneOffset; import java.util.stream.Stream; import org.apache.james.blob.api.BlobId; -import org.apache.james.blob.api.BlobIdEntropy; import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.BlobStoreCacheCallback; import org.apache.james.blob.api.BlobStoreDAO; @@ -40,12 +39,10 @@ import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.PlainBlobId; import org.apache.james.server.blob.deduplication.GenerationAwareBlobId; import org.apache.james.server.blob.deduplication.MinIOGenerationAwareBlobId; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import com.google.common.io.BaseEncoding; import com.google.common.io.ByteSource; import reactor.core.publisher.Mono; @@ -62,7 +59,6 @@ import reactor.util.function.Tuple2; class ContentRecoveryMessageContentSaverTest { private static final byte[] HEADER_BYTES = "Subject: test\r\n\r\n".getBytes(StandardCharsets.UTF_8); private static final ByteSource BODY = ByteSource.wrap("body".getBytes(StandardCharsets.UTF_8)); - private static final BaseEncoding BLOB_ID_ENCODING = BaseEncoding.base64Url().omitPadding(); /** * 2026-09-04T00:00:00Z is exactly 690 times the default 30 days generation duration, which keeps the @@ -111,15 +107,6 @@ class ContentRecoveryMessageContentSaverTest { .isNotEqualTo(saveContent(blobIdFactory).getT1()); } - @Test - void headerBlobIdShouldDrawTheConfiguredEntropy() { - String headerBlobId = saveContent(new PlainBlobId.Factory()).getT1().asString(); - String randomPart = headerBlobId.substring(0, headerBlobId.length() - HEADER_BLOB_ID_SUFFIX.length()); - - assertThat(BLOB_ID_ENCODING.decode(randomPart)) - .hasSize(BlobIdEntropy.entropyBytes()); - } - private Tuple2<BlobId, BlobId> saveContent(BlobId.Factory blobIdFactory) { BlobStoreDAO blobStoreDAO = mock(BlobStoreDAO.class); when(blobStoreDAO.save(any(BucketName.class), any(BlobId.class), any(BlobStoreDAO.Blob.class))).thenReturn(Mono.empty()); diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobId.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobId.java index 815907914f..93917db2c2 100644 --- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobId.java +++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobId.java @@ -25,7 +25,19 @@ public interface BlobId { BlobId of(String id); BlobId parse(String id); + + BlobIdEncoding encoding(); + + default BlobId random() { + return of(encoding().encode(BlobIdEntropy.randomBytes())); + } + + default BlobId ofHash(byte[] hash) { + return of(encoding().encode(BlobIdEntropy.truncate(hash))); + } } String asString(); + + BlobId withSuffix(String suffix); } diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEncoding.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEncoding.java new file mode 100644 index 0000000000..ecceb81c41 --- /dev/null +++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEncoding.java @@ -0,0 +1,71 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.blob.api; + +import java.util.Optional; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.io.BaseEncoding; + +/** + * How the payload of a blob id is spelled out, as set by the {@code james.blob.id.hash.encoding} system + * property. + * + * <p>Truncated ids are left unpadded: they exist to be short, and padding them back up would give away + * part of what {@link BlobIdEntropy} saved. Ids at full entropy keep the padding of their encoding, so + * that ids of existing deployments are left untouched.</p> + */ +public class BlobIdEncoding { + public static final String ENCODING_PROPERTY = "james.blob.id.hash.encoding"; + private static final BaseEncoding DEFAULT_ENCODING = BaseEncoding.base64Url(); + + public static BlobIdEncoding fromSystemProperties() { + return new BlobIdEncoding(Optional.ofNullable(System.getProperty(ENCODING_PROPERTY)) + .map(BlobIdEncoding::baseEncodingFrom) + .orElse(DEFAULT_ENCODING)); + } + + @VisibleForTesting + static BaseEncoding baseEncodingFrom(String encodingType) { + return switch (encodingType) { + case "base16", "hex" -> BaseEncoding.base16(); + case "base32" -> BaseEncoding.base32(); + case "base32Hex" -> BaseEncoding.base32Hex(); + case "base64" -> BaseEncoding.base64(); + case "base64Url" -> BaseEncoding.base64Url(); + default -> throw new IllegalArgumentException("Unknown encoding type: " + encodingType); + }; + } + + private final BaseEncoding encoding; + + @VisibleForTesting + BlobIdEncoding(BaseEncoding encoding) { + if (BlobIdEntropy.entropyBits() == BlobIdEntropy.DEFAULT_ENTROPY_BITS) { + this.encoding = encoding; + } else { + this.encoding = encoding.omitPadding(); + } + } + + public String encode(byte[] payload) { + return encoding.encode(payload); + } +} diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/PlainBlobId.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/PlainBlobId.java index 0022827252..d285d9fb68 100644 --- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/PlainBlobId.java +++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/PlainBlobId.java @@ -24,6 +24,15 @@ import com.google.common.base.Strings; public record PlainBlobId(String id) implements BlobId { public static class Factory implements BlobId.Factory { + private final BlobIdEncoding encoding; + + public Factory() { + this(BlobIdEncoding.fromSystemProperties()); + } + + public Factory(BlobIdEncoding encoding) { + this.encoding = encoding; + } @Override public PlainBlobId of(String id) { @@ -35,10 +44,20 @@ public record PlainBlobId(String id) implements BlobId { public PlainBlobId parse(String id) { return of(id); } + + @Override + public BlobIdEncoding encoding() { + return encoding; + } } @Override public String asString() { return id; } + + @Override + public PlainBlobId withSuffix(String suffix) { + return new PlainBlobId(id + suffix); + } } diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEncodingTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEncodingTest.java new file mode 100644 index 0000000000..6b3f983656 --- /dev/null +++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEncodingTest.java @@ -0,0 +1,82 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.blob.api; + +import static org.apache.james.blob.api.BlobIdEncoding.ENCODING_PROPERTY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import com.google.common.io.BaseEncoding; + +class BlobIdEncodingTest { + private static final byte[] PAYLOAD = "payload".getBytes(StandardCharsets.UTF_8); + + @BeforeEach + @AfterEach + void clearProperty() { + System.clearProperty(ENCODING_PROPERTY); + } + + @Test + void blobIdFactoryCreationShouldFailOnInvalidProperty() { + System.setProperty(ENCODING_PROPERTY, "blobIdFactoryCreationShouldFailOnInvalidProperty"); + + assertThatThrownBy(PlainBlobId.Factory::new) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unknown encoding type: blobIdFactoryCreationShouldFailOnInvalidProperty"); + } + + @ParameterizedTest + @ValueSource(strings = {"base16", "hex", "base32", "base32Hex", "base64", "base64Url"}) + void blobIdFactoryCreationShouldAcceptSupportedEncodings(String encoding) { + System.setProperty(ENCODING_PROPERTY, encoding); + + assertThatCode(PlainBlobId.Factory::new).doesNotThrowAnyException(); + } + + @Test + void shouldDefaultToBase64Url() { + assertThat(BlobIdEncoding.fromSystemProperties().encode(PAYLOAD)) + .isEqualTo(BaseEncoding.base64Url().encode(PAYLOAD)); + } + + @Test + void shouldHonourTheConfiguredEncoding() { + System.setProperty(ENCODING_PROPERTY, "base16"); + + assertThat(BlobIdEncoding.fromSystemProperties().encode(PAYLOAD)) + .isEqualTo(BaseEncoding.base16().encode(PAYLOAD)); + } + + @Test + void hexShouldBeAnAliasOfBase16() { + assertThat(BlobIdEncoding.baseEncodingFrom("hex")) + .isEqualTo(BlobIdEncoding.baseEncodingFrom("base16")); + } +} diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java index cd4fd2ee58..ab20cdf5a5 100644 --- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java +++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java @@ -24,14 +24,10 @@ import static org.apache.james.blob.api.BlobStore.StoragePolicy.LOW_COST; import static org.apache.james.blob.api.BlobStore.StoragePolicy.SIZE_BASED; import static org.apache.james.blob.api.BlobStoreContract.SHORT_BYTEARRAY; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.ByteArrayInputStream; import java.util.stream.Stream; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -54,25 +50,6 @@ public interface DeduplicationBlobStoreContract { BlobStore createBlobStore(); - @BeforeEach - default void beforeEach() { - System.clearProperty("james.blob.id.hash.encoding"); - } - - @AfterEach - default void afterEach() { - System.clearProperty("james.blob.id.hash.encoding"); - } - - @Test - default void deduplicationBlobstoreCreationShouldFailOnInvalidProperty() { - System.setProperty("james.blob.id.hash.encoding", "deduplicationBlobstoreCreationShouldFailOnInvalidProperty"); - - assertThatThrownBy(this::createBlobStore) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Unknown encoding type: deduplicationBlobstoreCreationShouldFailOnInvalidProperty"); - } - @ParameterizedTest @MethodSource("storagePolicies") default void saveShouldReturnBlobIdOfString(BlobStore.StoragePolicy storagePolicy) { diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/PlainBlobIdTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/PlainBlobIdTest.java index da991b0d28..d555deefac 100644 --- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/PlainBlobIdTest.java +++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/PlainBlobIdTest.java @@ -22,11 +22,17 @@ package org.apache.james.blob.api; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.Arrays; + import org.junit.jupiter.api.Test; +import com.google.common.io.BaseEncoding; + import nl.jqno.equalsverifier.EqualsVerifier; class PlainBlobIdTest { + private static final BaseEncoding ENCODING = BaseEncoding.base64Url(); + private static final PlainBlobId.Factory BLOB_ID_FACTORY = new PlainBlobId.Factory(); @@ -53,4 +59,43 @@ class PlainBlobIdTest { assertThatThrownBy(() -> BLOB_ID_FACTORY.parse("")) .isInstanceOf(IllegalArgumentException.class); } + + @Test + void randomShouldDrawTheConfiguredEntropy() { + assertThat(ENCODING.decode(BLOB_ID_FACTORY.random().asString())) + .hasSize(BlobIdEntropy.entropyBytes()); + } + + @Test + void randomShouldNotRepeatItself() { + assertThat(BLOB_ID_FACTORY.random()) + .isNotEqualTo(BLOB_ID_FACTORY.random()); + } + + @Test + void ofHashShouldTruncateToTheConfiguredEntropy() { + byte[] hash = new byte[BlobIdEntropy.entropyBytes() + 8]; + + assertThat(ENCODING.decode(BLOB_ID_FACTORY.ofHash(hash).asString())) + .hasSize(BlobIdEntropy.entropyBytes()); + } + + @Test + void ofHashShouldKeepTheLeadingBytesOfTheHash() { + byte[] hash = new byte[BlobIdEntropy.entropyBytes() + 8]; + for (int i = 0; i < hash.length; i++) { + hash[i] = (byte) i; + } + + assertThat(ENCODING.decode(BLOB_ID_FACTORY.ofHash(hash).asString())) + .isEqualTo(Arrays.copyOf(hash, BlobIdEntropy.entropyBytes())); + } + + @Test + void ofHashShouldBeContentAddressed() { + byte[] hash = new byte[] {1, 2, 3}; + + assertThat(BLOB_ID_FACTORY.ofHash(hash)) + .isEqualTo(BLOB_ID_FACTORY.ofHash(hash)); + } } diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/TestBlobId.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/TestBlobId.java index 26e398eb38..59b7a7d1be 100644 --- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/TestBlobId.java +++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/TestBlobId.java @@ -22,6 +22,8 @@ package org.apache.james.blob.api; import java.util.Objects; public class TestBlobId implements BlobId { + private static final BlobIdEncoding ENCODING = BlobIdEncoding.fromSystemProperties(); + public static class Factory implements BlobId.Factory { @Override @@ -33,6 +35,11 @@ public class TestBlobId implements BlobId { public BlobId parse(String id) { return of(id); } + + @Override + public BlobIdEncoding encoding() { + return ENCODING; + } } private final String rawValue; @@ -46,6 +53,11 @@ public class TestBlobId implements BlobId { return rawValue; } + @Override + public TestBlobId withSuffix(String suffix) { + return new TestBlobId(rawValue + suffix); + } + @Override public final boolean equals(Object o) { if (o instanceof TestBlobId) { diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/GenerationAwareBlobId.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/GenerationAwareBlobId.java index 67fd492b16..7f370d6b2c 100644 --- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/GenerationAwareBlobId.java +++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/GenerationAwareBlobId.java @@ -27,6 +27,7 @@ import java.util.Objects; import java.util.Optional; import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.BlobIdEncoding; import org.apache.james.util.DurationParser; import com.google.common.annotations.VisibleForTesting; @@ -143,6 +144,11 @@ public class GenerationAwareBlobId implements BlobId, GenerationAware { return new GenerationAwareBlobId(generation, family, wrapped); } + @Override + public BlobIdEncoding encoding() { + return delegate.encoding(); + } + private GenerationAwareBlobId decorateWithoutGeneration(String id) { return new GenerationAwareBlobId(NO_GENERATION, NO_FAMILY, delegate.parse(id)); } @@ -182,6 +188,11 @@ public class GenerationAwareBlobId implements BlobId, GenerationAware { return family + "_" + generation + "_" + delegate.asString(); } + @Override + public GenerationAwareBlobId withSuffix(String suffix) { + return new GenerationAwareBlobId(generation, family, delegate.withSuffix(suffix)); + } + @Override public boolean inActiveGeneration(Configuration configuration, Instant now) { return configuration.getFamily() == this.family && diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/MinIOGenerationAwareBlobId.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/MinIOGenerationAwareBlobId.java index c612e17e53..dc34fedb1e 100644 --- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/MinIOGenerationAwareBlobId.java +++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/MinIOGenerationAwareBlobId.java @@ -29,6 +29,7 @@ import java.util.Objects; import jakarta.inject.Inject; import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.BlobIdEncoding; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; @@ -73,6 +74,11 @@ public class MinIOGenerationAwareBlobId implements BlobId, GenerationAware { } } + @Override + public BlobIdEncoding encoding() { + return delegate.encoding(); + } + private static String injectFoldersInBlobId(String blobIdPart) { int folderDepthToCreate = 2; if (blobIdPart.length() > folderDepthToCreate) { @@ -120,6 +126,11 @@ public class MinIOGenerationAwareBlobId implements BlobId, GenerationAware { return family + "/" + generation + "/" + delegate.asString(); } + @Override + public MinIOGenerationAwareBlobId withSuffix(String suffix) { + return new MinIOGenerationAwareBlobId(generation, family, delegate.withSuffix(suffix)); + } + @Override public boolean inActiveGeneration(GenerationAwareBlobId.Configuration configuration, Instant now) { return configuration.getFamily() == this.family && diff --git a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala index bec9312852..202d400767 100644 --- a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala +++ b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala @@ -20,13 +20,13 @@ package org.apache.james.server.blob.deduplication import com.google.common.base.Preconditions -import com.google.common.hash.{HashCode, Hashing, HashingInputStream} -import com.google.common.io.{BaseEncoding, ByteSource, FileBackedOutputStream} +import com.google.common.hash.{Hashing, HashingInputStream} +import com.google.common.io.{ByteSource, FileBackedOutputStream} import jakarta.inject.{Inject, Named} import org.apache.commons.io.IOUtils import org.apache.james.blob.api.BlobStore.BlobIdProvider import org.apache.james.blob.api.BlobStoreDAO.{ByteSourceBlob, BytesBlob, InputStreamBlob} -import org.apache.james.blob.api.{BlobId, BlobIdEntropy, BlobStore, BlobStoreDAO, BucketName} +import org.apache.james.blob.api.{BlobId, BlobStore, BlobStoreDAO, BucketName} import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore.THREAD_SWITCH_THRESHOLD import org.reactivestreams.Publisher import reactor.core.publisher.{Flux, Mono} @@ -43,34 +43,12 @@ object DeDuplicationBlobStore { val FILE_THRESHOLD = Integer.parseInt(System.getProperty("james.deduplicating.blobstore.file.threshold", "10240")) val THREAD_SWITCH_THRESHOLD = Integer.parseInt(System.getProperty("james.deduplicating.blobstore.thread.switch.threshold", "32768")); - private def baseEncodingFrom(encodingType: String): BaseEncoding = encodingType match { - case "base16" => - BaseEncoding.base16 - case "hex" => - BaseEncoding.base16 - case "base64" => - BaseEncoding.base64 - case "base64Url" => - BaseEncoding.base64Url - case "base32" => - BaseEncoding.base32 - case "base32Hex" => - BaseEncoding.base32Hex - case _ => - throw new IllegalArgumentException("Unknown encoding type: " + encodingType) - } } class DeDuplicationBlobStore @Inject()(blobStoreDAO: BlobStoreDAO, @Named(BlobStore.DEFAULT_BUCKET_NAME_QUALIFIER) defaultBucketName: BucketName, blobIdFactory: BlobId.Factory) extends BlobStore { - private val HASH_BLOB_ID_ENCODING_TYPE_PROPERTY = "james.blob.id.hash.encoding" - private val HASH_BLOB_ID_ENCODING_DEFAULT = BaseEncoding.base64Url - private val baseEncoding = Option(System.getProperty(HASH_BLOB_ID_ENCODING_TYPE_PROPERTY)).map(DeDuplicationBlobStore.baseEncodingFrom).getOrElse(HASH_BLOB_ID_ENCODING_DEFAULT) - // Truncated ids exist to be short: padding them back up would give away part of what was saved. - // Left untouched at full entropy so that ids of existing deployments are preserved. - private val blobIdEncoding = if (BlobIdEntropy.entropyBits() == BlobIdEntropy.DEFAULT_ENTROPY_BITS) baseEncoding else baseEncoding.omitPadding() override def save(bucketName: BucketName, data: Array[Byte], storagePolicy: BlobStore.StoragePolicy): Publisher[BlobId] = { save(bucketName, data, withBlobIdFromArray, storagePolicy) @@ -111,7 +89,7 @@ class DeDuplicationBlobStore @Inject()(blobStoreDAO: BlobStoreDAO, (fileBackedOutputStream: FileBackedOutputStream) => SMono.fromCallable(() => { IOUtils.copy(hashingInputStream, fileBackedOutputStream) - (blobIdFactory.of(base64(hashingInputStream.hash)), fileBackedOutputStream.asByteSource.openStream()) + (blobIdFactory.ofHash(hashingInputStream.hash.asBytes), fileBackedOutputStream.asByteSource.openStream()) }).asJava() Mono.using[(BlobId, InputStream),FileBackedOutputStream]( @@ -126,29 +104,21 @@ class DeDuplicationBlobStore @Inject()(blobStoreDAO: BlobStoreDAO, private def withBlobIdFromByteSource: BlobIdProvider[ByteSource] = data => Mono.fromCallable(() => data.hash(Hashing.sha256())) .subscribeOn(Schedulers.boundedElastic()) - .map(base64) - .map(blobIdFactory.of) + .map(hash => blobIdFactory.ofHash(hash.asBytes)) .map(blobId => Tuples.of(blobId, data)) private def withBlobIdFromArray: BlobIdProvider[Array[Byte]] = data => { if (data.length < THREAD_SWITCH_THRESHOLD) { - val code = Hashing.sha256.hashBytes(data) - val blobId = blobIdFactory.of(base64(code)) + val blobId = blobIdFactory.ofHash(Hashing.sha256.hashBytes(data).asBytes) Mono.just(Tuples.of(blobId, data)) } else { SMono.fromCallable(() => { - val code = Hashing.sha256.hashBytes(data) - val blobId = blobIdFactory.of(base64(code)) + val blobId = blobIdFactory.ofHash(Hashing.sha256.hashBytes(data).asBytes) Tuples.of(blobId, data) }) } } - private def base64(hashCode: HashCode) = { - val bytes = BlobIdEntropy.truncate(hashCode.asBytes) - blobIdEncoding.encode(bytes) - } - override def save(bucketName: BucketName, data: InputStream, blobIdProvider: BlobIdProvider[InputStream], diff --git a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala index 9d5c677b4c..a5493e8bd0 100644 --- a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala +++ b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala @@ -20,7 +20,6 @@ package org.apache.james.server.blob.deduplication import java.io.InputStream -import java.util.UUID import com.google.common.base.Preconditions import com.google.common.io.ByteSource @@ -88,11 +87,11 @@ class PassThroughBlobStore @Inject()(blobStoreDAO: BlobStoreDAO, } private def withBlobId: BlobIdProvider[InputStream] = data => - SMono.just(Tuples.of(blobIdFactory.of(UUID.randomUUID.toString), data)) + SMono.just(Tuples.of(blobIdFactory.random(), data)) private def withBlobIdByteArray: BlobIdProvider[Array[Byte]] = data => - SMono.just(Tuples.of(blobIdFactory.of(UUID.randomUUID.toString), data)) + SMono.just(Tuples.of(blobIdFactory.random(), data)) private def withBlobIdByteSource: BlobIdProvider[ByteSource] = data => - SMono.just(Tuples.of(blobIdFactory.of(UUID.randomUUID.toString), data)) + SMono.just(Tuples.of(blobIdFactory.random(), data)) override def readBytes(bucketName: BucketName, blobId: BlobId): Publisher[Array[Byte]] = { Preconditions.checkNotNull(bucketName) diff --git a/server/mailrepository/mailrepository-blob/src/main/scala/org/apache/james/mailrepository/blob/BlobMailRepositoryFactory.scala b/server/mailrepository/mailrepository-blob/src/main/scala/org/apache/james/mailrepository/blob/BlobMailRepositoryFactory.scala index a0208ffdff..976e8691b8 100644 --- a/server/mailrepository/mailrepository-blob/src/main/scala/org/apache/james/mailrepository/blob/BlobMailRepositoryFactory.scala +++ b/server/mailrepository/mailrepository-blob/src/main/scala/org/apache/james/mailrepository/blob/BlobMailRepositoryFactory.scala @@ -37,6 +37,10 @@ class MailRepositoryBlobIdFactory( override def of(id: String): BlobId = blobIdFactory.of(url.getPath.subPath(id).asString()) + // Random and content addressed ids go through `of`, hence through the url prefix, on their own. + override def encoding(): BlobIdEncoding = + blobIdFactory.encoding() + } class BlobMailRepositoryFactory(blobStoreDao: BlobStoreDAO, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
