This is an automated email from the ASF dual-hosted git repository.
quantranhong1999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new 59fd1791b1 JAMES-4222 Imrpove Ceph/Rados support with
?allow-unordered=true
59fd1791b1 is described below
commit 59fd1791b1c6ac4fa52e10521ce892f1dc6a83ee
Author: Benoit TELLIER <[email protected]>
AuthorDate: Fri Sep 4 09:49:04 2026 +0200
JAMES-4222 Imrpove Ceph/Rados support with ?allow-unordered=true
---
.../servers/partials/configure/blobstore.adoc | 13 ++++
docs/modules/servers/partials/configure/jvm.adoc | 15 ++++
.../blob/objectstorage/aws/S3ClientFactory.java | 4 +
.../aws/UnorderedListingInterceptor.java | 52 +++++++++++++
.../aws/UnorderedListingInterceptorTest.java | 87 ++++++++++++++++++++++
5 files changed, 171 insertions(+)
diff --git a/docs/modules/servers/partials/configure/blobstore.adoc
b/docs/modules/servers/partials/configure/blobstore.adoc
index 44d992c36e..feb4726c1d 100644
--- a/docs/modules/servers/partials/configure/blobstore.adoc
+++ b/docs/modules/servers/partials/configure/blobstore.adoc
@@ -203,6 +203,19 @@ To enable blob hierarchy compatible with MinIO add in
`jvm.properties`:
james.s3.minio.compatibility.mode=true
----
+==== Unordered listing for Ceph RADOS Gateway
+
+Ceph RADOS Gateway supports an `allow-unordered` extension on bucket listings:
instead of merging the entries of
+every bucket shard in order, the gateway returns them as they come, which is
significantly cheaper on sharded
+buckets. James never relies on the ordering of blob listings, thus this is a
safe trade for RADOS backed
+deployments. Other S3 implementations might reject the extra parameter, hence
it is disabled by default.
+
+To let RADOS answer bucket listings in an unordered fashion add in
`jvm.properties`:
+
+----
+james.s3.rados.allow.unorder=true
+----
+
== Blob Export
diff --git a/docs/modules/servers/partials/configure/jvm.adoc
b/docs/modules/servers/partials/configure/jvm.adoc
index b1116d7701..76cfa72e55 100644
--- a/docs/modules/servers/partials/configure/jvm.adoc
+++ b/docs/modules/servers/partials/configure/jvm.adoc
@@ -103,6 +103,21 @@ james.s3.sdk.checksum.backward.compatibility=false
----
To disable S3 checksum backward compatibility.
+== Unordered S3 listings
+
+Ceph RADOS Gateway supports an `allow-unordered` extension on bucket listings:
instead of merging the entries of
+every bucket shard in order, the gateway returns them as they come, which is
significantly cheaper on sharded
+buckets. James never relies on the ordering of blob listings, thus this is a
safe trade for RADOS backed
+deployments. Other S3 implementations might reject the extra parameter, hence
it is disabled by default.
+
+Optional. Boolean. Defaults to false.
+
+Ex in `jvm.properties`
+----
+james.s3.rados.allow.unorder=true
+----
+To let RADOS answer bucket listings in an unordered fashion.
+
endif::[]
ifndef::no-jmap[]
== JMAP Quota draft compatibility
diff --git
a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3ClientFactory.java
b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3ClientFactory.java
index 5ea18bd128..9815aaf250 100644
---
a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3ClientFactory.java
+++
b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3ClientFactory.java
@@ -77,6 +77,7 @@ public class S3ClientFactory implements Startable, Closeable {
public static final String S3_METRICS_ENABLED_DEFAULT_VALUE = "true";
public static final String S3_METRICS_PREFIX =
System.getProperty("james.s3.metrics.prefix", DEFAULT_S3_METRICS_PREFIX);
public static final boolean S3_CHECKSUM_BACKWARD_COMPATIBILITY_ENABLED =
Boolean.parseBoolean(System.getProperty("james.s3.sdk.checksum.backward.compatibility",
"true"));
+ public static final boolean S3_RADOS_ALLOW_UNORDER =
Boolean.parseBoolean(System.getProperty("james.s3.rados.allow.unorder",
"false"));
private final S3AsyncClient s3Client;
@@ -107,6 +108,9 @@ public class S3ClientFactory implements Startable,
Closeable {
if (s3MetricsEnabled) {
builder.addMetricPublisher(jamesS3MetricPublisherProvider.get());
}
+ if (S3_RADOS_ALLOW_UNORDER) {
+ builder.addExecutionInterceptor(new
UnorderedListingInterceptor());
+ }
});
if (S3_CHECKSUM_BACKWARD_COMPATIBILITY_ENABLED) {
diff --git
a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/UnorderedListingInterceptor.java
b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/UnorderedListingInterceptor.java
new file mode 100644
index 0000000000..599e2c39d3
--- /dev/null
+++
b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/UnorderedListingInterceptor.java
@@ -0,0 +1,52 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws;
+
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+
+/**
+ * Ceph RADOS Gateway supports an {@code allow-unordered} extension on bucket
listings: rather than merging the
+ * results of every bucket shard in order, the gateway returns the entries of
each shard as they come, which is
+ * significantly cheaper on sharded buckets.
+ *
+ * James never relies on the ordering of {@link
org.apache.james.blob.api.BlobStoreDAO#listBlobs} results, thus
+ * unordered listing is a safe trade for deployments backed by RADOS.
+ *
+ * Note that RADOS rejects {@code allow-unordered} combined with a {@code
delimiter}: James never sets one.
+ */
+public class UnorderedListingInterceptor implements ExecutionInterceptor {
+ private static final String ALLOW_UNORDERED = "allow-unordered";
+
+ @Override
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
ExecutionAttributes executionAttributes) {
+ if (context.request() instanceof ListObjectsV2Request ||
context.request() instanceof ListObjectsRequest) {
+ return context.httpRequest()
+ .toBuilder()
+ .putRawQueryParameter(ALLOW_UNORDERED, "true")
+ .build();
+ }
+ return context.httpRequest();
+ }
+}
diff --git
a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/UnorderedListingInterceptorTest.java
b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/UnorderedListingInterceptorTest.java
new file mode 100644
index 0000000000..ef7efb7fde
--- /dev/null
+++
b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/UnorderedListingInterceptorTest.java
@@ -0,0 +1,87 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.net.URI;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.InterceptorContext;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+
+class UnorderedListingInterceptorTest {
+ private static final SdkHttpRequest HTTP_REQUEST = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.GET)
+ .uri(URI.create("http://localhost:8080/bucket";))
+ .build();
+
+ private final UnorderedListingInterceptor testee = new
UnorderedListingInterceptor();
+
+ private SdkHttpRequest modify(SdkRequest request) {
+ return testee.modifyHttpRequest(InterceptorContext.builder()
+ .request(request)
+ .httpRequest(HTTP_REQUEST)
+ .build(),
+ new ExecutionAttributes());
+ }
+
+ @Test
+ void shouldAllowUnorderedListingForListObjectsV2() {
+
assertThat(modify(ListObjectsV2Request.builder().bucket("bucket").build()).rawQueryParameters())
+ .containsEntry("allow-unordered", List.of("true"));
+ }
+
+ @Test
+ void shouldAllowUnorderedListingForListObjects() {
+
assertThat(modify(ListObjectsRequest.builder().bucket("bucket").build()).rawQueryParameters())
+ .containsEntry("allow-unordered", List.of("true"));
+ }
+
+ @Test
+ void shouldNotAlterNonListingRequests() {
+
assertThat(modify(GetObjectRequest.builder().bucket("bucket").key("key").build()).rawQueryParameters())
+ .isEmpty();
+ }
+
+ @Test
+ void shouldNotDuplicateTheParameterWhenAlreadyPresent() {
+ SdkHttpRequest alreadySet = HTTP_REQUEST.toBuilder()
+ .putRawQueryParameter("allow-unordered", "true")
+ .build();
+
+ SdkHttpRequest result =
testee.modifyHttpRequest(InterceptorContext.builder()
+
.request(ListObjectsV2Request.builder().bucket("bucket").build())
+ .httpRequest(alreadySet)
+ .build(),
+ new ExecutionAttributes());
+
+ assertThat(result.rawQueryParameters())
+ .containsEntry("allow-unordered", List.of("true"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
