This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-10656-atomic-key-overwrite
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to
refs/heads/HDDS-10656-atomic-key-overwrite by this push:
new f862049d1a HDDS-10921. Enable Atomic Rewrite in FSO buckets (#6740)
f862049d1a is described below
commit f862049d1a1d675fd95d7b5125bf8594d00384a2
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Tue May 28 21:23:38 2024 +0100
HDDS-10921. Enable Atomic Rewrite in FSO buckets (#6740)
---
.../client/rpc/TestOzoneRpcClientAbstract.java | 10 ++++-----
.../ozone/om/request/key/OMKeyCommitRequest.java | 2 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 8 +++++++-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 1 +
.../om/request/key/TestOMKeyCommitRequest.java | 17 +++++++--------
.../request/key/TestOMKeyCommitRequestWithFSO.java | 24 ++++++++++++++--------
.../om/request/key/TestOMKeyCreateRequest.java | 24 ++++++----------------
.../request/key/TestOMKeyCreateRequestWithFSO.java | 5 +++++
9 files changed, 50 insertions(+), 43 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 18e8d281fe..e7e24b17ac 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -1101,7 +1101,7 @@ public abstract class TestOzoneRpcClientAbstract extends
OzoneTestBase {
}
@ParameterizedTest
- @EnumSource(names = { "OBJECT_STORE" }) // TODO - only works on object store
layout for now.
+ @EnumSource
void rewriteKey(BucketLayout layout) throws IOException {
OzoneBucket bucket = createBucket(layout);
OzoneKeyDetails keyDetails = createTestKey(bucket);
@@ -1115,7 +1115,7 @@ public abstract class TestOzoneRpcClientAbstract extends
OzoneTestBase {
}
@ParameterizedTest
- @EnumSource(names = { "OBJECT_STORE" }) // TODO - only works on object store
layout for now.
+ @EnumSource
void overwriteAfterRewrite(BucketLayout layout) throws IOException {
OzoneBucket bucket = createBucket(layout);
OzoneKeyDetails keyDetails = createTestKey(bucket);
@@ -1130,7 +1130,7 @@ public abstract class TestOzoneRpcClientAbstract extends
OzoneTestBase {
}
@ParameterizedTest
- @EnumSource(names = { "OBJECT_STORE" }) // TODO - only works on object store
layout for now.
+ @EnumSource
void rewriteFailsDueToOutdatedGeneration(BucketLayout layout) throws
IOException {
OzoneBucket bucket = createBucket(layout);
OzoneKeyDetails keyDetails = createTestKey(bucket);
@@ -1147,7 +1147,7 @@ public abstract class TestOzoneRpcClientAbstract extends
OzoneTestBase {
}
@ParameterizedTest
- @EnumSource(names = { "OBJECT_STORE" }) // TODO - only works on object store
layout for now.
+ @EnumSource
void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws
IOException {
OzoneBucket bucket = createBucket(layout);
OzoneKeyDetails keyDetails = createTestKey(bucket);
@@ -1177,7 +1177,7 @@ public abstract class TestOzoneRpcClientAbstract extends
OzoneTestBase {
}
@ParameterizedTest
- @EnumSource(names = { "OBJECT_STORE" }) // TODO - only works on object store
layout for now.
+ @EnumSource
void cannotRewriteDeletedKey(BucketLayout layout) throws IOException {
OzoneBucket bucket = createBucket(layout);
OzoneKeyDetails keyDetails = createTestKey(bucket);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 7b27db67d4..7d31422c80 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -504,7 +504,7 @@ public class OMKeyCommitRequest extends OMKeyRequest {
return req;
}
- private void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit,
Map<String, String> auditMap)
+ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit,
Map<String, String> auditMap)
throws OMException {
if (toCommit.getExpectedDataGeneration() != null) {
// These values are not passed in the request keyArgs, so add them into
the auditMap if they are present
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index 704e9e91c4..d2cd3fd5ce 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -148,7 +148,7 @@ public class OMKeyCommitRequestWithFSO extends
OMKeyCommitRequest {
action = "hsync";
}
throw new OMException("Failed to " + action + " key, as " +
- dbOpenFileKey + "entry is not found in the OpenKey table",
+ dbOpenFileKey + " entry is not found in the OpenKey table",
KEY_NOT_FOUND);
}
omKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf(
@@ -176,6 +176,12 @@ public class OMKeyCommitRequestWithFSO extends
OMKeyCommitRequest {
Map<String, RepeatedOmKeyInfo> oldKeyVersionsToDeleteMap = null;
OmKeyInfo keyToDelete =
omMetadataManager.getKeyTable(getBucketLayout()).get(dbFileKey);
+
+ validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap);
+ // Optimistic locking validation has passed. Now set the rewrite fields
to null so they are
+ // not persisted in the key table.
+ omKeyInfo.setExpectedDataGeneration(null);
+
if (null != keyToDelete) {
final String clientIdString
= String.valueOf(commitKeyRequest.getClientID());
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index f323916b79..36f2a8c31b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -442,7 +442,7 @@ public class OMKeyCreateRequest extends OMKeyRequest {
return req;
}
- private void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs)
+ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs)
throws OMException {
if (keyArgs.hasExpectedDataGeneration()) {
// If a key does not exist, or if it exists but the updateID do not
match, then fail this request.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
index 6fe8c12085..d8d9b19343 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
@@ -120,6 +120,7 @@ public class OMKeyCreateRequestWithFSO extends
OMKeyCreateRequest {
dbFileInfo = OMFileRequest.getOmKeyInfoFromFileTable(false,
omMetadataManager, dbFileKey, keyName);
}
+ validateAtomicRewrite(dbFileInfo, keyArgs);
// Check if a file or directory exists with same key name.
if (pathInfoFSO.getDirectoryResult() == DIRECTORY_EXISTS) {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
index be70213513..0503578d99 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
@@ -226,11 +226,6 @@ public class TestOMKeyCommitRequest extends
TestOMKeyRequest {
@Test
public void testAtomicRewrite() throws Exception {
- if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- // TODO - does not work with in FSO for now
- return;
- }
-
Table<String, OmKeyInfo> openKeyTable =
omMetadataManager.getOpenKeyTable(getBucketLayout());
Table<String, OmKeyInfo> closedKeyTable =
omMetadataManager.getKeyTable(getBucketLayout());
@@ -255,9 +250,7 @@ public class TestOMKeyCommitRequest extends
TestOMKeyRequest {
List<OzoneAcl> acls =
Collections.singletonList(OzoneAcl.parseAcl("user:foo:rw"));
omKeyInfo.addAcl(acls.get(0));
- String openKey = getOzonePathKey() + "/" +
modifiedOmRequest.getCommitKeyRequest().getClientID();
-
- openKeyTable.put(openKey, omKeyInfo);
+ String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfo);
OmKeyInfo openKeyInfo = openKeyTable.get(openKey);
assertNotNull(openKeyInfo);
assertEquals(acls, openKeyInfo.getAcls());
@@ -827,6 +820,14 @@ public class TestOMKeyCommitRequest extends
TestOMKeyRequest {
keyName, clientID);
}
+ @Nonnull
+ protected String addKeyToOpenKeyTable(List<OmKeyLocationInfo> locationList,
OmKeyInfo keyInfo) throws Exception {
+ OMRequestTestUtils.addKeyToTable(true, false, keyInfo, clientID, 0,
omMetadataManager);
+
+ return omMetadataManager.getOpenKey(volumeName, bucketName,
+ keyName, clientID);
+ }
+
@Nonnull
protected OMKeyCommitRequest getOmKeyCommitRequest(OMRequest omRequest) {
return new OMKeyCommitRequest(omRequest, BucketLayout.DEFAULT);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestWithFSO.java
index 48cc52773a..c3b913b825 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestWithFSO.java
@@ -69,30 +69,36 @@ public class TestOMKeyCommitRequestWithFSO extends
TestOMKeyCommitRequest {
}
@Override
- protected String addKeyToOpenKeyTable(List<OmKeyLocationInfo> locationList)
+ protected String addKeyToOpenKeyTable(List<OmKeyLocationInfo> locationList,
OmKeyInfo keyInfo)
throws Exception {
// need to initialize parentID
if (getParentDir() == null) {
parentID = getBucketID();
} else {
parentID = OMRequestTestUtils.addParentsToDirTable(volumeName,
- bucketName, getParentDir(), omMetadataManager);
+ bucketName, getParentDir(), omMetadataManager);
}
+ keyInfo.setParentObjectID(parentID);
+ keyInfo.appendNewBlocks(locationList, false);
+
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ return OMRequestTestUtils.addFileToKeyTable(true, false,
+ fileName, keyInfo, clientID, txnLogId, omMetadataManager);
+
+ }
+
+ @Override
+ protected String addKeyToOpenKeyTable(List<OmKeyLocationInfo> locationList)
+ throws Exception {
long objectId = 100;
OmKeyInfo omKeyInfoFSO =
OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
RatisReplicationConfig.getInstance(ONE), new
OmKeyLocationInfoGroup(version, new ArrayList<>(), false))
.setObjectID(objectId)
- .setParentObjectID(parentID)
.setUpdateID(100L)
.build();
- omKeyInfoFSO.appendNewBlocks(locationList, false);
-
- String fileName = OzoneFSUtils.getFileName(keyName);
- return OMRequestTestUtils.addFileToKeyTable(true, false,
- fileName, omKeyInfoFSO, clientID, txnLogId, omMetadataManager);
-
+ return addKeyToOpenKeyTable(locationList, omKeyInfoFSO);
}
@Nonnull
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
index ebaefbcfde..1bc141a22e 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.om.request.key;
import java.io.IOException;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -931,11 +930,6 @@ public class TestOMKeyCreateRequest extends
TestOMKeyRequest {
when(ozoneManager.getOzoneLockProvider()).thenReturn(
new OzoneLockProvider(setKeyPathLock, setFileSystemPaths));
- if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- // TODO: Test is not applicable for FSO layout.
- return;
- }
-
OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, omMetadataManager,
OmBucketInfo.newBuilder().setVolumeName(volumeName)
.setBucketName(bucketName)
@@ -956,11 +950,10 @@ public class TestOMKeyCreateRequest extends
TestOMKeyRequest {
List<OzoneAcl> acls =
Collections.singletonList(OzoneAcl.parseAcl("user:foo:rw"));
OmKeyInfo createdKeyInfo = createAndCheck(keyName, metadata, acls);
// Commit openKey entry.
- OMRequestTestUtils.addKeyToTable(false, false,
- createdKeyInfo, 0L, 0L, omMetadataManager);
+ omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(),
createdKeyInfo);
+
// Retrieve the committed key info
- String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
keyName);
- OmKeyInfo existingKeyInfo =
omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey);
+ OmKeyInfo existingKeyInfo =
omMetadataManager.getKeyTable(getBucketLayout()).get(getOzoneKey());
List<OzoneAcl> existingAcls = existingKeyInfo.getAcls();
assertEquals(acls, existingAcls);
@@ -981,10 +974,8 @@ public class TestOMKeyCreateRequest extends
TestOMKeyRequest {
response = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 105L);
assertEquals(OK, response.getOMResponse().getStatus());
- // Ensure the expectedDataGeneration is persisted in the open key table
- String openKey = omMetadataManager.getOpenKey(volumeName, bucketName,
- keyName, omRequest.getCreateKeyRequest().getClientID());
- OmKeyInfo openKeyInfo =
omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()).get(openKey);
+ OmKeyInfo openKeyInfo =
omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .get(getOpenKey(omRequest.getCreateKeyRequest().getClientID()));
assertEquals(existingKeyInfo.getGeneration(),
openKeyInfo.getExpectedDataGeneration());
// Creation time should remain the same on rewrite.
@@ -1080,9 +1071,6 @@ public class TestOMKeyCreateRequest extends
TestOMKeyRequest {
OMKeyCreateRequest omKeyCreateRequest, OMRequest omRequest,
String keyName) throws Exception {
keyName = omKeyCreateRequest.validateAndNormalizeKey(true, keyName);
- // Check intermediate directories created or not.
- Path keyPath = Paths.get(keyName);
- checkIntermediatePaths(keyPath);
// Check open key entry
String openKey = omMetadataManager.getOpenKey(volumeName, bucketName,
@@ -1116,7 +1104,7 @@ public class TestOMKeyCreateRequest extends
TestOMKeyRequest {
}
protected OMKeyCreateRequest getOMKeyCreateRequest(OMRequest omRequest) {
- return new OMKeyCreateRequest(omRequest, BucketLayout.DEFAULT);
+ return new OMKeyCreateRequest(omRequest, getBucketLayout());
}
protected OMKeyCreateRequest getOMKeyCreateRequest(
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java
index 87badb2812..a5181b25a0 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java
@@ -153,6 +153,11 @@ public class TestOMKeyCreateRequestWithFSO extends
TestOMKeyCreateRequest {
long lastKnownParentId = omBucketInfo.getObjectID();
final long volumeId = omMetadataManager.getVolumeId(volumeName);
+ if (keyPath == null) {
+ // The file is at the root of the bucket, so it has no parent folder.
The parent is
+ // the bucket itself.
+ return lastKnownParentId;
+ }
Iterator<Path> elements = keyPath.iterator();
StringBuilder fullKeyPath = new StringBuilder(bucketKey);
while (elements.hasNext()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
