This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c1b27a87b5 HDDS-10630. Add missing parent directories deleted between
initiate and complete MPU (#6496)
c1b27a87b5 is described below
commit c1b27a87b57442f8c765fcd7e60802d72a16a932
Author: SaketaChalamchala <[email protected]>
AuthorDate: Thu Apr 11 23:40:31 2024 -0700
HDDS-10630. Add missing parent directories deleted between initiate and
complete MPU (#6496)
---
.../S3MultipartUploadCompleteRequest.java | 104 +++++++++++++++++++--
.../S3MultipartUploadCompleteRequestWithFSO.java | 39 +++++++-
.../S3MultipartUploadCompleteResponse.java | 8 ++
.../S3MultipartUploadCompleteResponseWithFSO.java | 45 ++++++++-
.../s3/multipart/TestS3MultipartResponse.java | 5 +-
5 files changed, 190 insertions(+), 11 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index 83b46de7fd..5a2359560a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.nio.file.InvalidPathException;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -31,6 +32,10 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.ozone.om.OzoneConfigUtil;
+import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequestWithFSO;
+import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
+import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
@@ -41,6 +46,7 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -176,11 +182,72 @@ public class S3MultipartUploadCompleteRequest extends
OMKeyRequest {
OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager,
volumeName, bucketName);
- String ozoneKey = omMetadataManager.getOzoneKey(
- volumeName, bucketName, keyName);
-
- String dbOzoneKey =
- getDBOzoneKey(omMetadataManager, volumeName, bucketName, keyName);
+ List<OmDirectoryInfo> missingParentInfos;
+ OMFileRequest.OMPathInfoWithFSO pathInfoFSO = OMFileRequest
+ .verifyDirectoryKeysInPath(omMetadataManager, volumeName, bucketName,
+ keyName, Paths.get(keyName));
+ missingParentInfos = OMDirectoryCreateRequestWithFSO
+ .getAllMissingParentDirInfo(ozoneManager, keyArgs, omBucketInfo,
+ pathInfoFSO, trxnLogIndex);
+
+ if (missingParentInfos != null) {
+ final long volumeId = omMetadataManager.getVolumeId(volumeName);
+ final long bucketId = omMetadataManager.getBucketId(volumeName,
+ bucketName);
+
+ // add all missing parents to directory table
+ addMissingParentsToCache(omBucketInfo, missingParentInfos,
+ omMetadataManager, volumeId, bucketId, trxnLogIndex);
+
+ String multipartOpenKey = omMetadataManager
+ .getMultipartKey(volumeId, bucketId,
+ pathInfoFSO.getLastKnownParentId(),
+ pathInfoFSO.getLeafNodeName(),
+ keyArgs.getMultipartUploadID());
+
+ if (getOmKeyInfoFromOpenKeyTable(multipartOpenKey,
+ keyName, omMetadataManager) == null) {
+
+ final ReplicationConfig replicationConfig = OzoneConfigUtil
+ .resolveReplicationConfigPreference(keyArgs.getType(),
+ keyArgs.getFactor(), keyArgs.getEcReplicationConfig(),
+ omBucketInfo != null ?
+ omBucketInfo.getDefaultReplicationConfig() :
+ null, ozoneManager);
+
+ OmMultipartKeyInfo multipartKeyInfoFromArgs =
+ new OmMultipartKeyInfo.Builder()
+ .setUploadID(keyArgs.getMultipartUploadID())
+ .setCreationTime(keyArgs.getModificationTime())
+ .setReplicationConfig(replicationConfig)
+ .setObjectID(pathInfoFSO.getLeafNodeObjectId())
+ .setUpdateID(trxnLogIndex)
+ .setParentID(pathInfoFSO.getLastKnownParentId())
+ .build();
+
+ OmKeyInfo keyInfoFromArgs = new OmKeyInfo.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setCreationTime(keyArgs.getModificationTime())
+ .setModificationTime(keyArgs.getModificationTime())
+ .setReplicationConfig(replicationConfig)
+ .setOmKeyLocationInfos(Collections.singletonList(
+ new OmKeyLocationInfoGroup(0, new ArrayList<>(), true)))
+ .setAcls(getAclsForKey(keyArgs, omBucketInfo, pathInfoFSO,
+ ozoneManager.getPrefixManager()))
+ .setObjectID(pathInfoFSO.getLeafNodeObjectId())
+ .setUpdateID(trxnLogIndex)
+ .setFileEncryptionInfo(keyArgs.hasFileEncryptionInfo() ?
+ OMPBHelper.convert(keyArgs.getFileEncryptionInfo()) : null)
+ .setParentObjectID(pathInfoFSO.getLastKnownParentId())
+ .build();
+
+ // Add missing multi part info to open key table
+ addMultiPartToCache(omMetadataManager, multipartOpenKey,
+ pathInfoFSO, keyInfoFromArgs, trxnLogIndex);
+ }
+ }
String dbMultipartOpenKey =
getDBMultipartOpenKey(volumeName, bucketName, keyName, uploadID,
@@ -189,6 +256,12 @@ public class S3MultipartUploadCompleteRequest extends
OMKeyRequest {
OmMultipartKeyInfo multipartKeyInfo = omMetadataManager
.getMultipartInfoTable().get(multipartKey);
+ String ozoneKey = omMetadataManager.getOzoneKey(
+ volumeName, bucketName, keyName);
+
+ String dbOzoneKey =
+ getDBOzoneKey(omMetadataManager, volumeName, bucketName, keyName);
+
// Check for directory exists with same name for the LEGACY_FS,
// if it exists throw error.
checkDirectoryAlreadyExists(ozoneManager, omBucketInfo, keyName,
@@ -284,7 +357,7 @@ public class S3MultipartUploadCompleteRequest extends
OMKeyRequest {
omClientResponse =
getOmClientResponse(multipartKey, omResponse, dbMultipartOpenKey,
omKeyInfo, allKeyInfoToRemove, omBucketInfo,
- volumeId, bucketId);
+ volumeId, bucketId, missingParentInfos, multipartKeyInfo);
result = Result.SUCCESS;
} else {
@@ -325,7 +398,8 @@ public class S3MultipartUploadCompleteRequest extends
OMKeyRequest {
OMResponse.Builder omResponse, String dbMultipartOpenKey,
OmKeyInfo omKeyInfo, List<OmKeyInfo> allKeyInfoToRemove,
OmBucketInfo omBucketInfo,
- long volumeId, long bucketId) {
+ long volumeId, long bucketId, List<OmDirectoryInfo> missingParentInfos,
+ OmMultipartKeyInfo multipartKeyInfo) {
return new S3MultipartUploadCompleteResponse(omResponse.build(),
multipartKey, dbMultipartOpenKey, omKeyInfo, allKeyInfoToRemove,
@@ -464,6 +538,22 @@ public class S3MultipartUploadCompleteRequest extends
OMKeyRequest {
return omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
}
+ protected void addMissingParentsToCache(OmBucketInfo omBucketInfo,
+ List<OmDirectoryInfo> missingParentInfos,
+ OMMetadataManager omMetadataManager,
+ long volumeId, long bucketId, long transactionLogIndex
+ ) throws IOException {
+ // FSO is disabled. Do nothing.
+ }
+
+ protected void addMultiPartToCache(
+ OMMetadataManager omMetadataManager, String multipartOpenKey,
+ OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo,
+ long transactionLogIndex
+ ) throws IOException {
+ // FSO is disabled. Do nothing.
+ }
+
protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneKey,
String keyName, OMMetadataManager omMetadataManager) throws IOException {
return omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
index c224786b10..c13d2746ae 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
@@ -23,7 +23,9 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import
org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse;
@@ -74,6 +76,37 @@ public class S3MultipartUploadCompleteRequestWithFSO
}
}
+ @Override
+ protected void addMissingParentsToCache(OmBucketInfo omBucketInfo,
+ List<OmDirectoryInfo> missingParentInfos,
+ OMMetadataManager omMetadataManager, long volumeId, long bucketId,
+ long transactionLogIndex) throws IOException {
+
+ // validate and update namespace for missing parent directory.
+ checkBucketQuotaInNamespace(omBucketInfo, missingParentInfos.size());
+ omBucketInfo.incrUsedNamespace(missingParentInfos.size());
+
+ // Add cache entries for the missing parent directories.
+ OMFileRequest.addDirectoryTableCacheEntries(omMetadataManager,
+ volumeId, bucketId, transactionLogIndex,
+ missingParentInfos, null);
+ }
+
+ @Override
+ protected void addMultiPartToCache(
+ OMMetadataManager omMetadataManager, String multipartOpenKey,
+ OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo,
+ long transactionLogIndex
+ ) throws IOException {
+
+ // Add multi part to cache
+ OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager,
+ multipartOpenKey, omKeyInfo, pathInfoFSO.getLeafNodeName(),
+ transactionLogIndex);
+
+ }
+
+
@Override
protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneFileKey,
String keyName, OMMetadataManager omMetadataManager) throws IOException {
@@ -147,11 +180,13 @@ public class S3MultipartUploadCompleteRequestWithFSO
OzoneManagerProtocolProtos.OMResponse.Builder omResponse,
String dbMultipartOpenKey, OmKeyInfo omKeyInfo,
List<OmKeyInfo> allKeyInfoToRemove, OmBucketInfo omBucketInfo,
- long volumeId, long bucketId) {
+ long volumeId, long bucketId, List<OmDirectoryInfo> missingParentInfos,
+ OmMultipartKeyInfo multipartKeyInfo) {
return new S3MultipartUploadCompleteResponseWithFSO(omResponse.build(),
multipartKey, dbMultipartOpenKey, omKeyInfo, allKeyInfoToRemove,
- getBucketLayout(), omBucketInfo, volumeId, bucketId);
+ getBucketLayout(), omBucketInfo, volumeId, bucketId,
+ missingParentInfos, multipartKeyInfo);
}
@Override
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
index 9fb843dcbe..a1f7b796cd 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
@@ -129,4 +129,12 @@ public class S3MultipartUploadCompleteResponse extends
OmKeyResponse {
protected OmKeyInfo getOmKeyInfo() {
return omKeyInfo;
}
+
+ protected OmBucketInfo getOmBucketInfo() {
+ return omBucketInfo;
+ }
+
+ protected String getMultiPartKey() {
+ return multipartKey;
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java
index 8774627ee6..4d1a6ce09b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java
@@ -22,7 +22,9 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -52,6 +54,10 @@ public class S3MultipartUploadCompleteResponseWithFSO
private long volumeId;
private long bucketId;
+ private List<OmDirectoryInfo> missingParentInfos;
+
+ private OmMultipartKeyInfo multipartKeyInfo;
+
@SuppressWarnings("checkstyle:ParameterNumber")
public S3MultipartUploadCompleteResponseWithFSO(
@Nonnull OMResponse omResponse,
@@ -61,11 +67,15 @@ public class S3MultipartUploadCompleteResponseWithFSO
@Nonnull List<OmKeyInfo> allKeyInfoToRemove,
@Nonnull BucketLayout bucketLayout,
OmBucketInfo omBucketInfo,
- @Nonnull long volumeId, @Nonnull long bucketId) {
+ @Nonnull long volumeId, @Nonnull long bucketId,
+ List<OmDirectoryInfo> missingParentInfos,
+ OmMultipartKeyInfo multipartKeyInfo) {
super(omResponse, multipartKey, multipartOpenKey, omKeyInfo,
allKeyInfoToRemove, bucketLayout, omBucketInfo);
this.volumeId = volumeId;
this.bucketId = bucketId;
+ this.missingParentInfos = missingParentInfos;
+ this.multipartKeyInfo = multipartKeyInfo;
}
/**
@@ -78,6 +88,39 @@ public class S3MultipartUploadCompleteResponseWithFSO
checkStatusNotOK();
}
+ @Override
+ public void addToDBBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+ if (missingParentInfos != null) {
+ // Create missing parent directory entries.
+ for (OmDirectoryInfo parentDirInfo : missingParentInfos) {
+ final String parentKey = omMetadataManager.getOzonePathKey(
+ volumeId, bucketId, parentDirInfo.getParentObjectID(),
+ parentDirInfo.getName());
+ omMetadataManager.getDirectoryTable().putWithBatch(batchOperation,
+ parentKey, parentDirInfo);
+ }
+
+ // namespace quota changes for parent directory
+ String bucketKey = omMetadataManager.getBucketKey(
+ getOmBucketInfo().getVolumeName(),
+ getOmBucketInfo().getBucketName());
+ omMetadataManager.getBucketTable().putWithBatch(batchOperation,
+ bucketKey, getOmBucketInfo());
+
+ if (OMFileRequest.getOmKeyInfoFromFileTable(true,
+ omMetadataManager, getMultiPartKey(), getOmKeyInfo().getKeyName())
+ != null) {
+ // Add multi part to open key table.
+ OMFileRequest.addToOpenFileTableForMultipart(omMetadataManager,
+ batchOperation,
+ getOmKeyInfo(), multipartKeyInfo.getUploadID(), volumeId,
+ bucketId);
+ }
+ }
+ super.addToDBBatch(omMetadataManager, batchOperation);
+ }
+
@Override
protected String addToKeyTable(OMMetadataManager omMetadataManager,
BatchOperation batchOperation) throws IOException {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
index 51963a00a1..a2192ddb88 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
@@ -306,6 +306,8 @@ public class TestS3MultipartResponse {
String multipartKey = omMetadataManager
.getMultipartKey(volumeName, bucketName, keyName, multipartUploadID);
+ OmMultipartKeyInfo multipartKeyInfo = omMetadataManager
+ .getMultipartInfoTable().get(multipartKey);
final long volumeId = omMetadataManager.getVolumeId(volumeName);
final long bucketId = omMetadataManager.getBucketId(volumeName,
@@ -324,7 +326,8 @@ public class TestS3MultipartResponse {
return new S3MultipartUploadCompleteResponseWithFSO(omResponse,
multipartKey, multipartOpenKey, omKeyInfo, allKeyInfoToRemove,
- getBucketLayout(), omBucketInfo, volumeId, bucketId);
+ getBucketLayout(), omBucketInfo, volumeId, bucketId, null,
+ multipartKeyInfo);
}
protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
