This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new fa8bd9dd70 HDDS-12916. Support ETag in listObjects response (#8356)
fa8bd9dd70 is described below
commit fa8bd9dd70fb1d22df390b1bd265b5751bd4f60d
Author: Ivan Andika <[email protected]>
AuthorDate: Sat May 3 19:45:04 2025 +0800
HDDS-12916. Support ETag in listObjects response (#8356)
---
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 9 +-
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 152 ++++++++++++---------
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 133 +++++++++++++++++-
3 files changed, 226 insertions(+), 68 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index f1e26ed87e..bcb08f0c3d 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -23,6 +23,7 @@
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConsts.ETAG;
import static
org.apache.hadoop.ozone.OzoneConsts.MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD;
import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_MAXIMUM_ACCESS_ID_LENGTH;
@@ -1722,8 +1723,10 @@ public List<OzoneKey> listKeys(String volumeName, String
bucketName,
key.getCreationTime(),
key.getModificationTime(),
key.getReplicationConfig(),
+ Collections.singletonMap(ETAG, key.getETag()),
key.isFile(),
- key.getOwnerName()))
+ key.getOwnerName(),
+ Collections.emptyMap()))
.collect(Collectors.toList());
} else {
List<OmKeyInfo> keys = ozoneManagerClient.listKeys(
@@ -1735,8 +1738,10 @@ public List<OzoneKey> listKeys(String volumeName, String
bucketName,
key.getCreationTime(),
key.getModificationTime(),
key.getReplicationConfig(),
+ key.getMetadata(),
key.isFile(),
- key.getOwnerName()))
+ key.getOwnerName(),
+ key.getTags()))
.collect(Collectors.toList());
}
}
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index dc8f098286..cee69f0f36 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -98,7 +98,6 @@
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -373,9 +372,8 @@ public void testPutDoubleSlashPrefixObject() throws
IOException {
final String bucketName = getBucketName();
final String keyName = "//dir1";
final String content = "bar";
- OzoneConfiguration conf = cluster.getConf();
// Create a FSO bucket for test
- try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) {
+ try (OzoneClient ozoneClient = cluster.newClient()) {
ObjectStore store = ozoneClient.getObjectStore();
OzoneVolume volume = store.getS3Volume();
OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder()
@@ -502,8 +500,7 @@ public void testGetObjectWithoutETag() throws Exception {
String value = "sample value";
byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
- OzoneConfiguration conf = cluster.getConf();
- try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) {
+ try (OzoneClient ozoneClient = cluster.newClient()) {
ObjectStore store = ozoneClient.getObjectStore();
OzoneVolume volume = store.getS3Volume();
@@ -532,46 +529,16 @@ public void testGetObjectWithoutETag() throws Exception {
}
@Test
- public void testListObjectsMany() {
- final String bucketName = getBucketName();
- s3Client.createBucket(bucketName);
- final List<String> keyNames = Arrays.asList(
- getKeyName("1"),
- getKeyName("2"),
- getKeyName("3")
- );
-
- for (String keyName: keyNames) {
- s3Client.putObject(bucketName, keyName,
RandomStringUtils.secure().nextAlphanumeric(5));
- }
-
- ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
- .withBucketName(bucketName)
- .withMaxKeys(2);
- ObjectListing listObjectsResponse =
s3Client.listObjects(listObjectsRequest);
- assertThat(listObjectsResponse.getObjectSummaries()).hasSize(2);
- assertEquals(bucketName, listObjectsResponse.getBucketName());
- assertEquals(listObjectsResponse.getObjectSummaries().stream()
- .map(S3ObjectSummary::getKey).collect(Collectors.toList()),
- keyNames.subList(0, 2));
- assertTrue(listObjectsResponse.isTruncated());
-
-
- listObjectsRequest = new ListObjectsRequest()
- .withBucketName(bucketName)
- .withMaxKeys(2)
- .withMarker(listObjectsResponse.getNextMarker());
- listObjectsResponse = s3Client.listObjects(listObjectsRequest);
- assertThat(listObjectsResponse.getObjectSummaries()).hasSize(1);
- assertEquals(bucketName, listObjectsResponse.getBucketName());
- assertEquals(listObjectsResponse.getObjectSummaries().stream()
- .map(S3ObjectSummary::getKey).collect(Collectors.toList()),
- keyNames.subList(2, keyNames.size()));
- assertFalse(listObjectsResponse.isTruncated());
+ public void testListObjectsMany() throws Exception {
+ testListObjectsMany(false);
}
@Test
- public void testListObjectsManyV2() {
+ public void testListObjectsManyV2() throws Exception {
+ testListObjectsMany(true);
+ }
+
+ private void testListObjectsMany(boolean isListV2) throws Exception {
final String bucketName = getBucketName();
s3Client.createBucket(bucketName);
final List<String> keyNames = Arrays.asList(
@@ -579,34 +546,91 @@ public void testListObjectsManyV2() {
getKeyName("2"),
getKeyName("3")
);
+ final List<String> keyNamesWithoutETag = Arrays.asList(
+ getKeyName("4"),
+ getKeyName("5")
+ );
+ final Map<String, String> keyToEtag = new HashMap<>();
for (String keyName: keyNames) {
- s3Client.putObject(bucketName, keyName,
RandomStringUtils.secure().nextAlphanumeric(5));
+ PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName,
+ RandomStringUtils.secure().nextAlphanumeric(5));
+ keyToEtag.put(keyName, putObjectResult.getETag());
}
+ try (OzoneClient ozoneClient = cluster.newClient()) {
+ ObjectStore store = ozoneClient.getObjectStore();
- ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
- .withBucketName(bucketName)
- .withMaxKeys(2);
- ListObjectsV2Result listObjectsResponse =
s3Client.listObjectsV2(listObjectsRequest);
- assertThat(listObjectsResponse.getObjectSummaries()).hasSize(2);
- assertEquals(bucketName, listObjectsResponse.getBucketName());
- assertEquals(listObjectsResponse.getObjectSummaries().stream()
+ OzoneVolume volume = store.getS3Volume();
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ for (String keyNameWithoutETag : keyNamesWithoutETag) {
+ byte[] valueBytes =
RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8);
+ try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag,
+ valueBytes.length,
+ ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS,
ReplicationFactor.ONE),
+ Collections.emptyMap())) {
+ out.write(valueBytes);
+ }
+ }
+ }
+
+ List<S3ObjectSummary> objectSummaries;
+ String continuationToken;
+ if (isListV2) {
+ ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
+ .withBucketName(bucketName)
+ .withMaxKeys(2);
+ ListObjectsV2Result listObjectsResponse =
s3Client.listObjectsV2(listObjectsRequest);
+ objectSummaries = listObjectsResponse.getObjectSummaries();
+ assertEquals(bucketName, listObjectsResponse.getBucketName());
+ assertTrue(listObjectsResponse.isTruncated());
+ continuationToken = listObjectsResponse.getNextContinuationToken();
+ } else {
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
+ .withBucketName(bucketName)
+ .withMaxKeys(2);
+ ObjectListing listObjectsResponse =
s3Client.listObjects(listObjectsRequest);
+ objectSummaries = listObjectsResponse.getObjectSummaries();
+ assertEquals(bucketName, listObjectsResponse.getBucketName());
+ assertTrue(listObjectsResponse.isTruncated());
+ continuationToken = listObjectsResponse.getNextMarker();
+ }
+ assertThat(objectSummaries).hasSize(2);
+ assertEquals(objectSummaries.stream()
.map(S3ObjectSummary::getKey).collect(Collectors.toList()),
keyNames.subList(0, 2));
- assertTrue(listObjectsResponse.isTruncated());
+ for (S3ObjectSummary objectSummary : objectSummaries) {
+ assertEquals(keyToEtag.get(objectSummary.getKey()),
objectSummary.getETag());
+ }
+ // Include both keys with and without ETag
+ if (isListV2) {
+ ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
+ .withBucketName(bucketName)
+ .withMaxKeys(5)
+ .withContinuationToken(continuationToken);
+ ListObjectsV2Result listObjectsResponse =
s3Client.listObjectsV2(listObjectsRequest);
+ objectSummaries = listObjectsResponse.getObjectSummaries();
+ assertEquals(bucketName, listObjectsResponse.getBucketName());
+ assertFalse(listObjectsResponse.isTruncated());
+ } else {
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
+ .withBucketName(bucketName)
+ .withMaxKeys(5)
+ .withMarker(continuationToken);
+ ObjectListing listObjectsResponse =
s3Client.listObjects(listObjectsRequest);
+ objectSummaries = listObjectsResponse.getObjectSummaries();
+ assertEquals(bucketName, listObjectsResponse.getBucketName());
+ assertFalse(listObjectsResponse.isTruncated());
+ }
- listObjectsRequest = new ListObjectsV2Request()
- .withBucketName(bucketName)
- .withMaxKeys(2)
- .withContinuationToken(listObjectsResponse.getNextContinuationToken());
- listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest);
- assertThat(listObjectsResponse.getObjectSummaries()).hasSize(1);
- assertEquals(bucketName, listObjectsResponse.getBucketName());
- assertEquals(listObjectsResponse.getObjectSummaries().stream()
- .map(S3ObjectSummary::getKey).collect(Collectors.toList()),
- keyNames.subList(2, keyNames.size()));
- assertFalse(listObjectsResponse.isTruncated());
+ assertThat(objectSummaries).hasSize(3);
+ assertEquals(keyNames.get(2), objectSummaries.get(0).getKey());
+ assertEquals(keyNamesWithoutETag.get(0), objectSummaries.get(1).getKey());
+ assertEquals(keyNamesWithoutETag.get(1), objectSummaries.get(2).getKey());
+ for (S3ObjectSummary objectSummary : objectSummaries) {
+ assertEquals(keyToEtag.get(objectSummary.getKey()),
objectSummary.getETag());
+ }
}
@Test
@@ -961,7 +985,7 @@ private boolean isBucketEmpty(Bucket bucket) {
}
private String getBucketName() {
- return getBucketName(null);
+ return getBucketName("");
}
private String getBucketName(String suffix) {
@@ -969,7 +993,7 @@ private String getBucketName(String suffix) {
}
private String getKeyName() {
- return getKeyName(null);
+ return getKeyName("");
}
private String getKeyName(String suffix) {
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 834580b6e8..7ef1342886 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -20,7 +20,9 @@
import static org.apache.hadoop.ozone.OzoneConsts.MB;
import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest;
import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
@@ -28,17 +30,29 @@
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.stream.Collectors;
import javax.xml.bind.DatatypeConverter;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.s3.S3ClientFactory;
import org.apache.hadoop.ozone.s3.S3GatewayService;
import org.apache.ozone.test.OzoneTestBase;
@@ -57,7 +71,12 @@
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.model.Tag;
import software.amazon.awssdk.services.s3.model.Tagging;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
@@ -133,6 +152,116 @@ public void testPutObject() {
assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"",
getObjectResponse.eTag());
}
+ @Test
+ public void testListObjectsMany() throws Exception {
+ testListObjectsMany(false);
+ }
+
+ @Test
+ public void testListObjectsManyV2() throws Exception {
+ testListObjectsMany(true);
+ }
+
+ private void testListObjectsMany(boolean isListV2) throws Exception {
+ final String bucketName = getBucketName();
+ s3Client.createBucket(b -> b.bucket(bucketName));
+ final List<String> keyNames = Arrays.asList(
+ getKeyName("1"),
+ getKeyName("2"),
+ getKeyName("3")
+ );
+ final List<String> keyNamesWithoutETag = Arrays.asList(
+ getKeyName("4"),
+ getKeyName("5")
+ );
+ final Map<String, String> keyToEtag = new HashMap<>();
+ for (String keyName: keyNames) {
+ PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
+ .bucket(bucketName)
+ .key(keyName),
+
RequestBody.fromString(RandomStringUtils.secure().nextAlphanumeric(5)));
+ keyToEtag.put(keyName, putObjectResponse.eTag());
+ }
+ try (OzoneClient ozoneClient = cluster.newClient()) {
+ ObjectStore store = ozoneClient.getObjectStore();
+
+ OzoneVolume volume = store.getS3Volume();
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ for (String keyNameWithoutETag : keyNamesWithoutETag) {
+ byte[] valueBytes =
RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8);
+ try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag,
+ valueBytes.length,
+ ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS,
ReplicationFactor.ONE),
+ Collections.emptyMap())) {
+ out.write(valueBytes);
+ }
+ }
+ }
+
+ List<S3Object> s3Objects;
+ String continuationToken;
+ if (isListV2) {
+ ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder()
+ .bucket(bucketName)
+ .maxKeys(2)
+ .build();
+ ListObjectsV2Response listObjectsResponse =
s3Client.listObjectsV2(listObjectsRequest);
+ s3Objects = listObjectsResponse.contents();
+ assertEquals(bucketName, listObjectsResponse.name());
+ assertTrue(listObjectsResponse.isTruncated());
+ continuationToken = listObjectsResponse.nextContinuationToken();
+ } else {
+ ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder()
+ .bucket(bucketName)
+ .maxKeys(2)
+ .build();
+ ListObjectsResponse listObjectsResponse =
s3Client.listObjects(listObjectsRequest);
+ s3Objects = listObjectsResponse.contents();
+ assertEquals(bucketName, listObjectsResponse.name());
+ assertTrue(listObjectsResponse.isTruncated());
+ continuationToken = listObjectsResponse.nextMarker();
+ }
+ assertThat(s3Objects).hasSize(2);
+ assertEquals(s3Objects.stream()
+ .map(S3Object::key).collect(Collectors.toList()),
+ keyNames.subList(0, 2));
+ for (S3Object s3Object : s3Objects) {
+ assertEquals(keyToEtag.get(s3Object.key()), s3Object.eTag());
+ }
+
+ // Include both keys with and without ETag
+ if (isListV2) {
+ ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder()
+ .bucket(bucketName)
+ .maxKeys(5)
+ .continuationToken(continuationToken)
+ .build();
+ ListObjectsV2Response listObjectsResponse =
s3Client.listObjectsV2(listObjectsRequest);
+ s3Objects = listObjectsResponse.contents();
+ assertEquals(bucketName, listObjectsResponse.name());
+ assertFalse(listObjectsResponse.isTruncated());
+ } else {
+ ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder()
+ .bucket(bucketName)
+ .maxKeys(5)
+ .marker(continuationToken)
+ .build();
+ ListObjectsResponse listObjectsResponse =
s3Client.listObjects(listObjectsRequest);
+ s3Objects = listObjectsResponse.contents();
+ assertEquals(bucketName, listObjectsResponse.name());
+ assertFalse(listObjectsResponse.isTruncated());
+ }
+
+ assertThat(s3Objects).hasSize(3);
+ assertEquals(keyNames.get(2), s3Objects.get(0).key());
+ assertEquals(keyNamesWithoutETag.get(0), s3Objects.get(1).key());
+ assertEquals(keyNamesWithoutETag.get(1), s3Objects.get(2).key());
+ for (S3Object s3Object : s3Objects) {
+ assertEquals(keyToEtag.get(s3Object.key()), s3Object.eTag());
+ }
+ }
+
@Test
public void testCopyObject() {
final String sourceBucketName = getBucketName("source");
@@ -196,7 +325,7 @@ public void testLowLevelMultipartUpload(@TempDir Path
tempDir) throws Exception
}
private String getBucketName() {
- return getBucketName(null);
+ return getBucketName("");
}
private String getBucketName(String suffix) {
@@ -204,7 +333,7 @@ private String getBucketName(String suffix) {
}
private String getKeyName() {
- return getKeyName(null);
+ return getKeyName("");
}
private String getKeyName(String suffix) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
