This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5f6306dd1d HDDS-10395. Fix eTag compatibility issues for MPU (#6235)
5f6306dd1d is described below
commit 5f6306dd1dde3c9c982ed933aa1fa8f4fa7a9301
Author: Ivan Andika <[email protected]>
AuthorDate: Thu Feb 22 18:47:58 2024 +0800
HDDS-10395. Fix eTag compatibility issues for MPU (#6235)
---
.../client/OzoneMultipartUploadPartListParts.java | 12 ++---
.../apache/hadoop/ozone/om/helpers/OmPartInfo.java | 24 ++++++----
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 14 ++++--
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 55 ++++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 11 ++++-
5 files changed, 97 insertions(+), 19 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadPartListParts.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadPartListParts.java
index 67f8edf314..c085720d19 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadPartListParts.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadPartListParts.java
@@ -98,13 +98,13 @@ public class OzoneMultipartUploadPartListParts {
/**
* Class that represents each Part information of a multipart upload part.
*/
- public static class PartInfo {
+ public static final class PartInfo {
- private int partNumber;
- private String partName;
- private long modificationTime;
- private long size;
- private String eTag;
+ private final int partNumber;
+ private final String partName;
+ private final long modificationTime;
+ private final long size;
+ private final String eTag;
public PartInfo(int number, String name, long time, long size,
String eTag) {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPartInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPartInfo.java
index e908c5a025..35d97cd4ff 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPartInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPartInfo.java
@@ -23,12 +23,12 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartInf
/**
* Class that defines information about each part of a multipart upload key.
*/
-public class OmPartInfo {
- private int partNumber;
- private String partName;
- private long modificationTime;
- private long size;
- private String eTag;
+public final class OmPartInfo {
+ private final int partNumber;
+ private final String partName;
+ private final long modificationTime;
+ private final long size;
+ private final String eTag;
public OmPartInfo(int number, String name, long time, long size,
String eTag) {
@@ -60,8 +60,14 @@ public class OmPartInfo {
}
public PartInfo getProto() {
- return
PartInfo.newBuilder().setPartNumber(partNumber).setPartName(partName)
- .setModificationTime(modificationTime)
- .setSize(size).setETag(eTag).build();
+ PartInfo.Builder builder = PartInfo.newBuilder()
+ .setPartNumber(partNumber)
+ .setPartName(partName)
+ .setModificationTime(modificationTime)
+ .setSize(size);
+ if (eTag != null) {
+ builder.setETag(eTag);
+ }
+ return builder.build();
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 2c9419e78d..3786601dd6 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -32,6 +32,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.Stack;
import java.util.TreeMap;
@@ -47,6 +48,7 @@ import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.storage.BlockLocationInfo;
@@ -822,13 +824,19 @@ public class KeyManagerImpl implements KeyManager {
if (nextPartNumberMarker > partNumberMarker) {
String partName = getPartName(partKeyInfo, volumeName, bucketName,
keyName);
+ // Before HDDS-9680, MPU part does not have eTag metadata, for
+ // this case, we return null. The S3G will handle this case by
+ // using the MPU part name as the eTag field instead.
+ Optional<HddsProtos.KeyValue> eTag = partKeyInfo.getPartKeyInfo()
+ .getMetadataList()
+ .stream()
+ .filter(keyValue -> keyValue.getKey().equals(ETAG))
+ .findFirst();
OmPartInfo omPartInfo = new OmPartInfo(partKeyInfo.getPartNumber(),
partName,
partKeyInfo.getPartKeyInfo().getModificationTime(),
partKeyInfo.getPartKeyInfo().getDataSize(),
- partKeyInfo.getPartKeyInfo().getMetadataList().stream()
- .filter(keyValue -> keyValue.getKey().equals(ETAG))
- .findFirst().get().getValue());
+ eTag.map(HddsProtos.KeyValue::getValue).orElse(null));
omPartInfoList.add(omPartInfo);
//if there are parts, use replication type from one of the parts
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
index 6454a77d66..278d96023c 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
@@ -161,6 +162,60 @@ class TestKeyManagerUnit extends OzoneTestBase {
omMultipartUploadListParts.getPartInfoList().size());
}
+ @Test
+ public void listMultipartUploadPartsWithoutEtagField() throws IOException {
+ // For backward compatibility reasons
+ final String volume = volumeName();
+ final String bucket = "bucketForEtag";
+ final String key = "dir/key1";
+ createBucket(metadataManager, volume, bucket);
+ OmMultipartInfo omMultipartInfo =
+ initMultipartUpload(writeClient, volume, bucket, key);
+
+
+ // Commit some MPU parts without eTag field
+ for (int i = 1; i <= 5; i++) {
+ OmKeyArgs partKeyArgs =
+ new OmKeyArgs.Builder()
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setKeyName(key)
+ .setIsMultipartKey(true)
+ .setMultipartUploadID(omMultipartInfo.getUploadID())
+ .setMultipartUploadPartNumber(i)
+ .setAcls(Collections.emptyList())
+ .setReplicationConfig(
+ RatisReplicationConfig.getInstance(ReplicationFactor.THREE))
+ .build();
+
+ OpenKeySession openKey = writeClient.openKey(partKeyArgs);
+
+ OmKeyArgs commitPartKeyArgs =
+ new OmKeyArgs.Builder()
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setKeyName(key)
+ .setIsMultipartKey(true)
+ .setMultipartUploadID(omMultipartInfo.getUploadID())
+ .setMultipartUploadPartNumber(i)
+ .setAcls(Collections.emptyList())
+ .setReplicationConfig(
+ RatisReplicationConfig.getInstance(ReplicationFactor.THREE))
+ .setLocationInfoList(Collections.emptyList())
+ .build();
+
+ writeClient.commitMultipartUploadPart(commitPartKeyArgs,
openKey.getId());
+ }
+
+
+ OmMultipartUploadListParts omMultipartUploadListParts = keyManager
+ .listParts(volume, bucket, key, omMultipartInfo.getUploadID(),
+ 0, 10);
+ assertEquals(5,
+ omMultipartUploadListParts.getPartInfoList().size());
+
+ }
+
private String volumeName() {
return getTestName();
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 4a36ad9e62..24115abe8e 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -999,6 +999,12 @@ public class ObjectEndpoint extends EndpointBase {
OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
keyOutputStream.getCommitUploadPartInfo();
String eTag = omMultipartCommitUploadPartInfo.getETag();
+ // If the OmMultipartCommitUploadPartInfo does not contain eTag,
+ // fall back to MPU part name for compatibility in case the (old) OM
+ // does not return the eTag field
+ if (StringUtils.isEmpty(eTag)) {
+ eTag = omMultipartCommitUploadPartInfo.getPartName();
+ }
if (copyHeader != null) {
getMetrics().updateCopyObjectSuccessStats(startNanos);
@@ -1069,7 +1075,10 @@ public class ObjectEndpoint extends EndpointBase {
ozoneMultipartUploadPartListParts.getPartInfoList().forEach(partInfo -> {
ListPartsResponse.Part part = new ListPartsResponse.Part();
part.setPartNumber(partInfo.getPartNumber());
- part.setETag(partInfo.getETag());
+ // If the ETag field does not exist, use MPU part name for backward
+ // compatibility
+ part.setETag(StringUtils.isNotEmpty(partInfo.getETag()) ?
+ partInfo.getETag() : partInfo.getPartName());
part.setSize(partInfo.getSize());
part.setLastModified(Instant.ofEpochMilli(
partInfo.getModificationTime()));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
