This is an automated email from the ASF dual-hosted git repository. dineshc pushed a commit to branch ozone-0.5.0 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 5c8f8e152dc7fabaf263a047652014f787d16bfe Author: Bharat Viswanadham <[email protected]> AuthorDate: Tue Mar 3 22:22:34 2020 -0800 HDDS-3069. UpdateID check should be skipped for non-HA OzoneManager. (#609) --- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 7 ++-- .../hadoop/ozone/om/helpers/WithObjectID.java | 39 +++++++++++++++++++--- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 8 ++--- .../om/request/bucket/OMBucketCreateRequest.java | 7 ++-- .../om/request/bucket/acl/OMBucketAclRequest.java | 3 +- .../ozone/om/request/file/OMFileCreateRequest.java | 3 +- .../om/request/key/OMAllocateBlockRequest.java | 2 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 2 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 3 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +-- .../ozone/om/request/key/OMKeyRenameRequest.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 4 +-- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +- .../om/request/key/acl/OMKeyAddAclRequest.java | 6 +--- .../request/key/acl/prefix/OMPrefixAclRequest.java | 2 +- .../multipart/S3MultipartUploadAbortRequest.java | 8 ++--- .../S3MultipartUploadCommitPartRequest.java | 10 +++--- .../S3MultipartUploadCompleteRequest.java | 2 +- .../om/request/volume/OMVolumeCreateRequest.java | 13 +++++--- .../om/request/volume/OMVolumeSetOwnerRequest.java | 3 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 3 +- .../om/request/volume/acl/OMVolumeAclRequest.java | 2 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 7 ++-- .../multipart/S3MultipartUploadAbortResponse.java | 6 ++-- .../S3MultipartUploadCommitPartResponse.java | 9 +++-- .../ozone/om/request/TestOMRequestUtils.java | 2 +- .../om/response/key/TestOMKeyDeleteResponse.java | 6 ++-- .../s3/multipart/TestS3MultipartResponse.java | 2 +- 28 files changed, 107 insertions(+), 61 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 47ee9ab..87522e3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -469,11 +469,10 @@ public final class OmUtils { * For regular Key deletes, this value should be set to * the same updaeID as is in keyInfo. * @return {@link RepeatedOmKeyInfo} - * @throws IOException if I/O Errors when checking for key */ public static RepeatedOmKeyInfo prepareKeyForDelete(OmKeyInfo keyInfo, - RepeatedOmKeyInfo repeatedOmKeyInfo, long trxnLogIndex) - throws IOException { + RepeatedOmKeyInfo repeatedOmKeyInfo, long trxnLogIndex, + boolean isRatisEnabled) { // If this key is in a GDPR enforced bucket, then before moving // KeyInfo to deletedTable, remove the GDPR related metadata and // FileEncryptionInfo from KeyInfo. @@ -485,7 +484,7 @@ public final class OmUtils { } // Set the updateID - keyInfo.setUpdateID(trxnLogIndex); + keyInfo.setUpdateID(trxnLogIndex, isRatisEnabled); if(repeatedOmKeyInfo == null) { //The key doesn't exist in deletedTable, so create a new instance. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java index 987bfd8..2526678 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java @@ -74,11 +74,42 @@ public class WithObjectID extends WithMetadata { * Sets the update ID. For each modification of this object, we will set * this to a value greater than the current value. * @param updateId long + * @param isRatisEnabled boolean */ - public void setUpdateID(long updateId) { - Preconditions.checkArgument(updateId >= this.updateID, String.format( - "Trying to set updateID to %d which is not greater than the current " + - "value of %d for %s", updateId, this.updateID, getObjectInfo())); + public void setUpdateID(long updateId, boolean isRatisEnabled) { + + // Because in non-HA, we have multiple rpc handler threads and + // transactionID is generated in OzoneManagerServerSideTranslatorPB. + + // Lets take T1 -> Set Bucket Property + // T2 -> Set Bucket Acl + + // Now T2 got lock first, so updateID will be set to 2. Now when T1 gets + // executed we will hit the precondition exception. So for OM non-HA with + // out ratis we should not have this check. + + // Same can happen after OM restart also. + + // OM Start + // T1 -> Create Bucket + // T2 -> Set Bucket Property + + // OM restart + // T1 -> Set Bucket Acl + + // So when T1 is executing, Bucket will have updateID 2 which is set by T2 + // execution before restart. + + // Main reason, in non-HA transaction Index after restart starts from 0. + // And also because of this same reason we don't do replay checks in non-HA. + + if (isRatisEnabled) { + Preconditions.checkArgument(updateId >= this.updateID, String.format( + "Trying to set updateID to %d which is not greater than the " + + "current value of %d for %s", updateId, this.updateID, + getObjectInfo())); + } + this.updateID = updateId; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index b066534..ca70906 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -820,7 +820,7 @@ public class KeyManagerImpl implements KeyManager { RepeatedOmKeyInfo repeatedOmKeyInfo = metadataManager.getDeletedTable().get(objectKey); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(keyInfo, - repeatedOmKeyInfo, 0L); + repeatedOmKeyInfo, 0L, false); metadataManager.getKeyTable().delete(objectKey); metadataManager.getDeletedTable().put(objectKey, repeatedOmKeyInfo); } catch (OMException ex) { @@ -1065,7 +1065,7 @@ public class KeyManagerImpl implements KeyManager { RepeatedOmKeyInfo repeatedOmKeyInfo = metadataManager.getDeletedTable().get(partName); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete( - keyInfo, repeatedOmKeyInfo, 0L); + keyInfo, repeatedOmKeyInfo, 0L, false); metadataManager.getDeletedTable().put(partName, repeatedOmKeyInfo); throw new OMException("No such Multipart upload is with specified " + "uploadId " + uploadID, ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); @@ -1102,7 +1102,7 @@ public class KeyManagerImpl implements KeyManager { .get(oldPartKeyInfo.getPartName()); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete( - partKey, repeatedOmKeyInfo, 0L); + partKey, repeatedOmKeyInfo, 0L, false); metadataManager.getDeletedTable().put(partName, repeatedOmKeyInfo); metadataManager.getDeletedTable().putWithBatch(batch, @@ -1220,7 +1220,7 @@ public class KeyManagerImpl implements KeyManager { .get(partKeyInfo.getPartName()); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete( - currentKeyPartInfo, repeatedOmKeyInfo, 0L); + currentKeyPartInfo, repeatedOmKeyInfo, 0L, false); metadataManager.getDeletedTable().putWithBatch(batch, partKeyInfo.getPartName(), repeatedOmKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index e14f40b..95ead23 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -185,9 +185,10 @@ public class OMBucketCreateRequest extends OMClientRequest { } // Add objectID and updateID - long objectId = OMFileRequest.getObjIDFromTxId(transactionLogIndex); - omBucketInfo.setObjectID(objectId); - omBucketInfo.setUpdateID(transactionLogIndex); + omBucketInfo.setObjectID( + OMFileRequest.getObjIDFromTxId(transactionLogIndex)); + omBucketInfo.setUpdateID(transactionLogIndex, + ozoneManager.isRatisEnabled()); // Add default acls from volume. addDefaultAcls(omBucketInfo, omVolumeArgs); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAclRequest.java index 06d403c..f162e88 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAclRequest.java @@ -117,7 +117,8 @@ public abstract class OMBucketAclRequest extends OMClientRequest { } operationResult = omBucketAclOp.apply(ozoneAcls, omBucketInfo); - omBucketInfo.setUpdateID(transactionLogIndex); + omBucketInfo.setUpdateID(transactionLogIndex, + ozoneManager.isRatisEnabled()); if (operationResult) { // update cache. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index ce14b63..5c58a58 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -298,7 +298,8 @@ public class OMFileCreateRequest extends OMKeyRequest { omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs, dbKeyInfo, keyArgs.getDataSize(), locations, encryptionInfo.orNull(), - ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex); + ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex, + ozoneManager.isRatisEnabled()); long openVersion = omKeyInfo.getLatestVersionLocations().getVersion(); long clientID = createFileRequest.getClientID(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 1c8b714..3bd305b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -218,7 +218,7 @@ public class OMAllocateBlockRequest extends OMKeyRequest { openKeyInfo.setModificationTime(keyArgs.getModificationTime()); // Set the UpdateID to current transactionLogIndex - openKeyInfo.setUpdateID(trxnLogIndex); + openKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // Add to cache. omMetadataManager.getOpenKeyTable().addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index caaa3ac..5f36a3c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -168,7 +168,7 @@ public class OMKeyCommitRequest extends OMKeyRequest { omKeyInfo.updateLocationInfoList(locationInfoList); // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // Add to cache of open key table and key table. omMetadataManager.getOpenKeyTable().addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 3715287..83100b6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -216,7 +216,8 @@ public class OMKeyCreateRequest extends OMKeyRequest { omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs, dbKeyInfo, keyArgs.getDataSize(), locations, encryptionInfo.orNull(), - ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex); + ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex, + ozoneManager.isRatisEnabled()); long openVersion = omKeyInfo.getLatestVersionLocations().getVersion(); long clientID = createKeyRequest.getClientID(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 0b5e907..f0e77cb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -144,7 +144,7 @@ public class OMKeyDeleteRequest extends OMKeyRequest { } // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // Update table cache. omMetadataManager.getKeyTable().addCacheEntry( @@ -159,7 +159,7 @@ public class OMKeyDeleteRequest extends OMKeyRequest { omClientResponse = new OMKeyDeleteResponse(omResponse .setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(), - omKeyInfo); + omKeyInfo, ozoneManager.isRatisEnabled()); result = Result.SUCCESS; } catch (IOException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index c92e097..ca97067 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -210,7 +210,7 @@ public class OMKeyRenameRequest extends OMKeyRequest { throw new OMException("Key not found " + fromKey, KEY_NOT_FOUND); } - fromKeyValue.setUpdateID(trxnLogIndex); + fromKeyValue.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); fromKeyValue.setKeyName(toKeyName); //Set modification time diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 248ffd1..34364c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -313,7 +313,7 @@ public abstract class OMKeyRequest extends OMClientRequest { @Nullable FileEncryptionInfo encInfo, @Nonnull PrefixManager prefixManager, @Nullable OmBucketInfo omBucketInfo, - long transactionLogIndex) + long transactionLogIndex, boolean isRatisEnabled) throws IOException { if (keyArgs.getIsMultipartKey()) { return prepareMultipartKeyInfo(omMetadataManager, keyArgs, @@ -332,7 +332,7 @@ public abstract class OMKeyRequest extends OMClientRequest { // The modification time is set in preExecute. Use the same // modification time. dbKeyInfo.setModificationTime(keyArgs.getModificationTime()); - dbKeyInfo.setUpdateID(transactionLogIndex); + dbKeyInfo.setUpdateID(transactionLogIndex, isRatisEnabled); return dbKeyInfo; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequest.java index 84634d4..025c258 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequest.java @@ -102,7 +102,7 @@ public abstract class OMKeyAclRequest extends OMClientRequest { } operationResult = apply(omKeyInfo, trxnLogIndex); - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // update cache. omMetadataManager.getKeyTable().addCacheEntry( @@ -187,6 +187,7 @@ public abstract class OMKeyAclRequest extends OMClientRequest { * Apply the acl operation, if successfully completed returns true, * else false. * @param omKeyInfo + * @param trxnLogIndex */ abstract boolean apply(OmKeyInfo omKeyInfo, long trxnLogIndex); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 9e18bf6..d0e0043 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -107,11 +107,7 @@ public class OMKeyAddAclRequest extends OMKeyAclRequest { @Override boolean apply(OmKeyInfo omKeyInfo, long trxnLogIndex) { // No need to check not null here, this will be never called with null. - boolean operationResult = omKeyInfo.addAcl(ozoneAcls.get(0)); - if (operationResult) { - omKeyInfo.setUpdateID(trxnLogIndex); - } - return operationResult; + return omKeyInfo.addAcl(ozoneAcls.get(0)); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAclRequest.java index 89c4057..7cde2c2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAclRequest.java @@ -106,7 +106,7 @@ public abstract class OMPrefixAclRequest extends OMClientRequest { } omPrefixInfo = operationResult.getOmPrefixInfo(); - omPrefixInfo.setUpdateID(trxnLogIndex); + omPrefixInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // As for remove acl list, for a prefix if after removing acl from // the existing acl list, if list size becomes zero, delete the diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 23f6dcd..bec1b48 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -145,7 +145,7 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest { multipartKeyInfo = omMetadataManager.getMultipartInfoTable() .get(multipartKey); - multipartKeyInfo.setUpdateID(trxnLogIndex); + multipartKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // Update cache of openKeyTable and multipartInfo table. // No need to add the cache entries to delete table, as the entries @@ -160,15 +160,15 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest { omClientResponse = new S3MultipartUploadAbortResponse( omResponse.setAbortMultiPartUploadResponse( MultipartUploadAbortResponse.newBuilder()).build(), - multipartKey, multipartKeyInfo); + multipartKey, multipartKeyInfo, ozoneManager.isRatisEnabled()); result = Result.SUCCESS; } catch (IOException ex) { result = Result.FAILURE; exception = ex; omClientResponse = - new S3MultipartUploadAbortResponse(createErrorOMResponse( - omResponse, exception), multipartKey, multipartKeyInfo); + new S3MultipartUploadAbortResponse(createErrorOMResponse(omResponse, + exception)); } finally { addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index e66c452..6ce2d27 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -162,7 +162,7 @@ public class S3MultipartUploadCommitPartRequest extends OMKeyRequest { // Set Modification time omKeyInfo.setModificationTime(keyArgs.getModificationTime()); // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); partName = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName) + clientID; @@ -192,7 +192,8 @@ public class S3MultipartUploadCommitPartRequest extends OMKeyRequest { multipartKeyInfo.addPartKeyInfo(partNumber, partKeyInfo.build()); // Set the UpdateID to current transactionLogIndex - multipartKeyInfo.setUpdateID(trxnLogIndex); + multipartKeyInfo.setUpdateID(trxnLogIndex, + ozoneManager.isRatisEnabled()); // OldPartKeyInfo will be deleted. Its updateID will be set in // S3MultipartUplodaCommitPartResponse before being added to @@ -218,7 +219,7 @@ public class S3MultipartUploadCommitPartRequest extends OMKeyRequest { .setPartName(partName)); omClientResponse = new S3MultipartUploadCommitPartResponse( omResponse.build(), multipartKey, openKey, omKeyInfo, - multipartKeyInfo, oldPartKeyInfo); + multipartKeyInfo, oldPartKeyInfo, ozoneManager.isRatisEnabled()); result = Result.SUCCESS; } catch (IOException ex) { @@ -231,7 +232,8 @@ public class S3MultipartUploadCommitPartRequest extends OMKeyRequest { exception = ex; omClientResponse = new S3MultipartUploadCommitPartResponse( createErrorOMResponse(omResponse, exception), multipartKey, - openKey, omKeyInfo, multipartKeyInfo, oldPartKeyInfo); + openKey, omKeyInfo, multipartKeyInfo, oldPartKeyInfo, + ozoneManager.isRatisEnabled()); } } finally { addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 1361bc2..1da81ba 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -274,7 +274,7 @@ public class S3MultipartUploadCompleteRequest extends OMKeyRequest { omKeyInfo.setModificationTime(keyArgs.getModificationTime()); omKeyInfo.setDataSize(dataSize); } - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); //Find all unused parts. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java index 9e17c09..47d8393 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java @@ -115,13 +115,16 @@ public class OMVolumeCreateRequest extends OMVolumeRequest { Map<String, String> auditMap = new HashMap<>(); Collection<String> ozAdmins = ozoneManager.getOzoneAdmins(); try { - long objectID = OMFileRequest.getObjIDFromTxId(transactionLogIndex); omVolumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo); - // when you create a volume, we set both Object ID and update ID to the - // same ratis transaction ID. The Object ID will never change, but update + // when you create a volume, we set both Object ID and update ID. + // The Object ID will never change, but update // ID will be set to transactionID each time we update the object. - omVolumeArgs.setUpdateID(transactionLogIndex); - omVolumeArgs.setObjectID(objectID); + omVolumeArgs.setObjectID( + OMFileRequest.getObjIDFromTxId(transactionLogIndex)); + omVolumeArgs.setUpdateID(transactionLogIndex, + ozoneManager.isRatisEnabled()); + + auditMap = omVolumeArgs.toAuditMap(); // check Acl diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java index 974b472..6b603e5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java @@ -159,7 +159,8 @@ public class OMVolumeSetOwnerRequest extends OMVolumeRequest { // Set owner with new owner name. omVolumeArgs.setOwnerName(newOwner); - omVolumeArgs.setUpdateID(transactionLogIndex); + omVolumeArgs.setUpdateID(transactionLogIndex, + ozoneManager.isRatisEnabled()); // Update cache. omMetadataManager.getUserTable().addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java index 5b28473..ff5ecd3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java @@ -131,7 +131,8 @@ public class OMVolumeSetQuotaRequest extends OMVolumeRequest { } omVolumeArgs.setQuotaInBytes(setVolumePropertyRequest.getQuotaInBytes()); - omVolumeArgs.setUpdateID(transactionLogIndex); + omVolumeArgs.setUpdateID(transactionLogIndex, + ozoneManager.isRatisEnabled()); // update cache. omMetadataManager.getVolumeTable().addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java index deab9e9..f2610e5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java @@ -108,7 +108,7 @@ public abstract class OMVolumeAclRequest extends OMClientRequest { // We set the updateID even if applyAcl = false to catch the replay // transactions. - omVolumeArgs.setUpdateID(trxnLogIndex); + omVolumeArgs.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // update cache. omMetadataManager.getVolumeTable().addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java index 6af8bc4..afff73a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java @@ -38,11 +38,13 @@ import javax.annotation.Nonnull; */ public class OMKeyDeleteResponse extends OMClientResponse { private OmKeyInfo omKeyInfo; + private boolean isRatisEnabled; public OMKeyDeleteResponse(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo) { + @Nonnull OmKeyInfo omKeyInfo, boolean isRatisEnabled) { super(omResponse); this.omKeyInfo = omKeyInfo; + this.isRatisEnabled = isRatisEnabled; } /** @@ -80,7 +82,8 @@ public class OMKeyDeleteResponse extends OMClientResponse { RepeatedOmKeyInfo repeatedOmKeyInfo = omMetadataManager.getDeletedTable().get(ozoneKey); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete( - omKeyInfo, repeatedOmKeyInfo, omKeyInfo.getUpdateID()); + omKeyInfo, repeatedOmKeyInfo, omKeyInfo.getUpdateID(), + isRatisEnabled); omMetadataManager.getDeletedTable().putWithBatch(batchOperation, ozoneKey, repeatedOmKeyInfo); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponse.java index e2654dc..606838f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponse.java @@ -42,13 +42,15 @@ public class S3MultipartUploadAbortResponse extends OMClientResponse { private String multipartKey; private OmMultipartKeyInfo omMultipartKeyInfo; + private boolean isRatisEnabled; public S3MultipartUploadAbortResponse(@Nonnull OMResponse omResponse, String multipartKey, - @Nonnull OmMultipartKeyInfo omMultipartKeyInfo) { + @Nonnull OmMultipartKeyInfo omMultipartKeyInfo, boolean isRatisEnabled) { super(omResponse); this.multipartKey = multipartKey; this.omMultipartKeyInfo = omMultipartKeyInfo; + this.isRatisEnabled = isRatisEnabled; } /** @@ -83,7 +85,7 @@ public class S3MultipartUploadAbortResponse extends OMClientResponse { omMetadataManager.getDeletedTable().get(partKeyInfo.getPartName()); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(currentKeyPartInfo, - repeatedOmKeyInfo, omMultipartKeyInfo.getUpdateID()); + repeatedOmKeyInfo, omMultipartKeyInfo.getUpdateID(), isRatisEnabled); omMetadataManager.getDeletedTable().putWithBatch(batchOperation, partKeyInfo.getPartName(), repeatedOmKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java index 330220f61..7ecfa97 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java @@ -47,19 +47,22 @@ public class S3MultipartUploadCommitPartResponse extends OMClientResponse { private OmKeyInfo deletePartKeyInfo; private OmMultipartKeyInfo omMultipartKeyInfo; private OzoneManagerProtocolProtos.PartKeyInfo oldMultipartKeyInfo; + private boolean isRatisEnabled; public S3MultipartUploadCommitPartResponse(@Nonnull OMResponse omResponse, String multipartKey, String openKey, @Nonnull OmKeyInfo deletePartKeyInfo, @Nonnull OmMultipartKeyInfo omMultipartKeyInfo, - @Nonnull OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo) { + @Nonnull OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo, + boolean isRatisEnabled) { super(omResponse); this.multipartKey = multipartKey; this.openKey = openKey; this.deletePartKeyInfo = deletePartKeyInfo; this.omMultipartKeyInfo = omMultipartKeyInfo; this.oldMultipartKeyInfo = oldPartKeyInfo; + this.isRatisEnabled = isRatisEnabled; } /** @@ -82,7 +85,7 @@ public class S3MultipartUploadCommitPartResponse extends OMClientResponse { omMetadataManager.getDeletedTable().get(openKey); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(deletePartKeyInfo, - repeatedOmKeyInfo, deletePartKeyInfo.getUpdateID()); + repeatedOmKeyInfo, deletePartKeyInfo.getUpdateID(), isRatisEnabled); omMetadataManager.getDeletedTable().putWithBatch(batchOperation, @@ -116,7 +119,7 @@ public class S3MultipartUploadCommitPartResponse extends OMClientResponse { .get(oldMultipartKeyInfo.getPartName()); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(partKey, - repeatedOmKeyInfo, omMultipartKeyInfo.getUpdateID()); + repeatedOmKeyInfo, omMultipartKeyInfo.getUpdateID(), isRatisEnabled); omMetadataManager.getDeletedTable().putWithBatch(batchOperation, oldMultipartKeyInfo.getPartName(), repeatedOmKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java index ee72e2e..82fa795 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java @@ -478,7 +478,7 @@ public final class TestOMRequestUtils { omMetadataManager.getDeletedTable().get(ozoneKey); repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(omKeyInfo, - repeatedOmKeyInfo, trxnLogIndex); + repeatedOmKeyInfo, trxnLogIndex, true); omMetadataManager.getDeletedTable().put(ozoneKey, repeatedOmKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java index 691df7e..f8b0a17 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java @@ -51,7 +51,7 @@ public class TestOMKeyDeleteResponse extends TestOMKeyResponse { .build(); OMKeyDeleteResponse omKeyDeleteResponse = new OMKeyDeleteResponse( - omResponse, omKeyInfo); + omResponse, omKeyInfo, true); String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); @@ -113,7 +113,7 @@ public class TestOMKeyDeleteResponse extends TestOMKeyResponse { .build(); OMKeyDeleteResponse omKeyDeleteResponse = new OMKeyDeleteResponse( - omResponse, omKeyInfo); + omResponse, omKeyInfo, true); Assert.assertTrue(omMetadataManager.getKeyTable().isExist(ozoneKey)); omKeyDeleteResponse.addToDBBatch(omMetadataManager, batchOperation); @@ -142,7 +142,7 @@ public class TestOMKeyDeleteResponse extends TestOMKeyResponse { .build(); OMKeyDeleteResponse omKeyDeleteResponse = new OMKeyDeleteResponse( - omResponse, omKeyInfo); + omResponse, omKeyInfo, true); String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java index edab04e..973783f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java @@ -117,7 +117,7 @@ public class TestS3MultipartResponse { MultipartUploadAbortResponse.newBuilder().build()).build(); return new S3MultipartUploadAbortResponse(omResponse, multipartKey, - omMultipartKeyInfo); + omMultipartKeyInfo, true); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
