This is an automated email from the ASF dual-hosted git repository. quantranhong1999 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 78d9e31704c9adf23c28bffde7341d732ac854f4 Author: Benoit TELLIER <[email protected]> AuthorDate: Mon Sep 7 17:19:00 2026 +0200 JAMES-4224 Allow for 96 bits entropy --- docs/modules/servers/partials/configure/jvm.adoc | 12 ++----- .../sample-configuration/jvm.properties | 4 +-- .../org/apache/james/blob/api/BlobIdEntropy.java | 19 +++++++--- .../apache/james/blob/api/BlobIdEntropyTest.java | 41 +++++++++++++++++++++- .../blob/api/DeduplicationBlobStoreContract.java | 7 +++- .../aws/S3WithMinIOGenerationAwareBlobIdTest.java | 2 +- 6 files changed, 66 insertions(+), 19 deletions(-) diff --git a/docs/modules/servers/partials/configure/jvm.adoc b/docs/modules/servers/partials/configure/jvm.adoc index 2ad1a7ab83..59b2c1fe01 100644 --- a/docs/modules/servers/partials/configure/jvm.adoc +++ b/docs/modules/servers/partials/configure/jvm.adoc @@ -110,7 +110,7 @@ Ex in `jvm.properties` james.blobid.entropy=256 ---- -Optional. Integer, a multiple of 8 within [128, 256]. Defaults to 128. +Optional. Integer, a multiple of 8 within [96, 256]. Defaults to 128. The length of an id is paid everywhere it is stored. The object key itself, but above all the Cassandra columns referencing it, which hold one id per message rather than one per blob: 256 bits takes a body @@ -120,13 +120,7 @@ more characters. 128 bits is ample for a content addressed store. The birthday bound puts a collision at `n^2/2^129`, ie. 1.5e-19 for ten billion blobs, twenty orders of magnitude below the silent error rate of the storage underneath; and truncating a cryptographic hash to its leading bits is standard practice (NIST SP -800-107, FIPS 180-4). Values below 128 bits are rejected: a collision in a deduplicated store means a -message silently inheriting the body of another. `256` spells ids out the way releases up to 3.9.x did. - -WARNING: This is an install time setting, not one to flip on a live deployment. Changing it loses -nothing, since ids are stored alongside the messages and existing blobs stay readable, but content -already stored under a differently spelled id will not deduplicate against its counterpart until it is -rewritten. +800-107, FIPS 180-4). `256` spells ids out the way releases up to 3.9.x did. == Improve listing support for MinIO @@ -333,4 +327,4 @@ Ex in `jvm.properties` james.mailbox.handleRecent=false ---- -Defaults to true (no breaking changes) \ No newline at end of file +Defaults to true (no breaking changes) diff --git a/server/apps/distributed-app/sample-configuration/jvm.properties b/server/apps/distributed-app/sample-configuration/jvm.properties index 29831fa8f4..dabfc953ae 100644 --- a/server/apps/distributed-app/sample-configuration/jvm.properties +++ b/server/apps/distributed-app/sample-configuration/jvm.properties @@ -102,12 +102,10 @@ jmx.remote.x.mlet.allow.getMBeansFromURL=false # james.jmap.preview.length=128 # Bits of entropy carried by a blobId: the SHA-256 is truncated to that many leading bits, and randomly -# generated ids draw that many. A multiple of 8 within [128, 256], defaults to 128. +# generated ids draw that many. A multiple of 8 within [96, 256], defaults to 128. # 256 spells ids out the way releases up to 3.9.x did, taking a body blobId from 28 to 50 chars in the # object key and in every Cassandra column referencing it, for no practical collision benefit: 128 bits # already puts a collision at 1.5e-19 for ten billion blobs. -# Install time setting: changing it on a live deployment stops new writes from deduplicating against -# blobs already stored under a differently spelled id. # james.blobid.entropy=256 # Count of octet from which hashing shall be done out of the IO threads in deduplicating blob store diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEntropy.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEntropy.java index da1e3a5dce..d88c4b96d3 100644 --- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEntropy.java +++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEntropy.java @@ -37,7 +37,8 @@ import com.google.common.base.Preconditions; public class BlobIdEntropy { public static final String ENTROPY_BITS_PROPERTY = "james.blobid.entropy"; public static final int DEFAULT_ENTROPY_BITS = 128; - private static final int MIN_ENTROPY_BITS = 128; + @VisibleForTesting + static final int MIN_ENTROPY_BITS = 96; /** The full SHA-256 output: the longest an id can usefully get. */ static final int MAX_ENTROPY_BITS = 256; private static final int BITS_PER_BYTE = 8; @@ -76,15 +77,25 @@ public class BlobIdEntropy { } public static byte[] randomBytes() { - byte[] bytes = new byte[entropyBytes()]; + return randomBytes(entropyBytes()); + } + + @VisibleForTesting + static byte[] randomBytes(int entropyBytes) { + byte[] bytes = new byte[entropyBytes]; SECURE_RANDOM.nextBytes(bytes); return bytes; } public static byte[] truncate(byte[] hash) { - if (hash.length <= entropyBytes()) { + return truncate(hash, entropyBytes()); + } + + @VisibleForTesting + static byte[] truncate(byte[] hash, int entropyBytes) { + if (hash.length <= entropyBytes) { return hash; } - return Arrays.copyOf(hash, entropyBytes()); + return Arrays.copyOf(hash, entropyBytes); } } diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEntropyTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEntropyTest.java index abd59ec5e9..febe3605a2 100644 --- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEntropyTest.java +++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEntropyTest.java @@ -22,8 +22,11 @@ package org.apache.james.blob.api; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import com.google.common.io.BaseEncoding; + class BlobIdEntropyTest { @Test void parseShouldReturnDefaultWhenNotSet() { @@ -64,7 +67,7 @@ class BlobIdEntropyTest { @Test void parseShouldRejectValueBelowTheSafetyFloor() { - assertThatThrownBy(() -> BlobIdEntropy.parse("96")) + assertThatThrownBy(() -> BlobIdEntropy.parse("88")) .isInstanceOf(IllegalArgumentException.class); } @@ -105,4 +108,40 @@ class BlobIdEntropyTest { assertThat(BlobIdEntropy.truncate(hash)).isEqualTo(hash); } + + @Nested + class NinetySixBits { + static final int ENTROPY_BYTES = 96 / 8; + // How the factory spells a truncated id: base64url, unpadded. + static final BaseEncoding ENCODING = BaseEncoding.base64Url().omitPadding(); + + @Test + void parseShouldAcceptTheSafetyFloor() { + assertThat(BlobIdEntropy.parse("96")).isEqualTo(BlobIdEntropy.MIN_ENTROPY_BITS); + } + + @Test + void randomBytesShouldDrawTwelveBytes() { + assertThat(BlobIdEntropy.randomBytes(ENTROPY_BYTES)).hasSize(ENTROPY_BYTES); + } + + @Test + void truncateShouldKeepTheTwelveLeadingBytesOfASha256() { + byte[] hash = new byte[32]; + for (int i = 0; i < hash.length; i++) { + hash[i] = (byte) i; + } + + assertThat(BlobIdEntropy.truncate(hash, ENTROPY_BYTES)) + .isEqualTo(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}); + } + + @Test + void idsShouldSpellOutSixteenCharacters() { + byte[] hash = new byte[32]; + + assertThat(ENCODING.encode(BlobIdEntropy.truncate(hash, ENTROPY_BYTES))) + .hasSize(16); + } + } } diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java index 9da5950986..69fa24586d 100644 --- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java +++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java @@ -32,6 +32,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import com.google.common.io.BaseEncoding; + import reactor.core.publisher.Mono; public interface DeduplicationBlobStoreContract { @@ -43,7 +45,10 @@ public interface DeduplicationBlobStoreContract { } String SHORT_STRING = "toto"; - String SHORT_STRING_BLOB_ID = "MfemXjFVhqwZi9eYtmKc5A"; + /** The SHA-256 of {@link #SHORT_STRING}, which every content addressed store spells its id from. */ + byte[] SHORT_STRING_HASH = BaseEncoding.base64Url().decode("MfemXjFVhqwZi9eYtmKc5JA9CJlHbVdBqfMuLlIbamY="); + /** That hash, truncated and spelled at the entropy the tests run with: "MfemXjFVhqwZi9eYtmKc5A" at the default 128 bits. */ + String SHORT_STRING_BLOB_ID = new BlobIdEncoding(BaseEncoding.base64Url()).encode(BlobIdEntropy.truncate(SHORT_STRING_HASH)); BlobStore testee(); diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3WithMinIOGenerationAwareBlobIdTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3WithMinIOGenerationAwareBlobIdTest.java index 1c7f7012ae..ef1e6b7dbf 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3WithMinIOGenerationAwareBlobIdTest.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3WithMinIOGenerationAwareBlobIdTest.java @@ -125,7 +125,7 @@ public class S3WithMinIOGenerationAwareBlobIdTest implements BlobStoreContract { BlobId blobId = Mono.from(store.save(defaultBucketName, "toto", storagePolicy)).block(); String blobIdString = blobId.asString(); - // Then: BlobId string and parsed BlobId should match expectations + // Then: BlobId string and parsed BlobId should match expectations, at the default 128 bits of entropy assertThat(blobIdString).isEqualTo("1/628/M/f/emXjFVhqwZi9eYtmKc5A"); assertThat(blobId).isEqualTo(blobIdFactory().parse(blobIdString)); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
