This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4578a06353 HDDS-8435. [Snapshot] Handle Directory renames for FSO
Buckets. (#4607)
4578a06353 is described below
commit 4578a063533bc1396a218a69613a842ff0b32ec6
Author: Aswin Shakil Balasubramanian <[email protected]>
AuthorDate: Tue May 2 14:23:57 2023 -0700
HDDS-8435. [Snapshot] Handle Directory renames for FSO Buckets. (#4607)
---
.../apache/hadoop/ozone/om/OMMetadataManager.java | 4 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 30 ++++----
.../hadoop/ozone/om/codec/OMDBDefinition.java | 18 ++---
.../ozone/om/response/key/OMKeyRenameResponse.java | 10 +--
.../response/key/OMKeyRenameResponseWithFSO.java | 29 ++++----
.../om/response/key/OMKeysRenameResponse.java | 8 +--
.../snapshot/OMSnapshotCreateResponse.java | 10 +--
.../OMSnapshotMoveDeletedKeysResponse.java | 2 +-
.../ozone/om/service/SnapshotDeletingService.java | 80 +++++++++++++++-------
.../snapshot/TestOMSnapshotCreateRequest.java | 70 +++++++++++++++----
.../om/response/key/TestOMKeyRenameResponse.java | 6 +-
11 files changed, 169 insertions(+), 98 deletions(-)
diff --git
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 7eed84f753..b3ce046c88 100644
---
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -370,7 +370,7 @@ public interface OMMetadataManager extends DBStoreHAManager
{
Table<String, SnapshotInfo> getSnapshotInfoTable();
- Table<String, String> getSnapshotRenamedKeyTable();
+ Table<String, String> getSnapshotRenamedTable();
/**
* Gets the OM Meta table.
@@ -492,7 +492,7 @@ public interface OMMetadataManager extends DBStoreHAManager
{
/**
* Given a volume, bucket and a objectID, return the DB key name in
- * snapshotRenamedKeyTable.
+ * snapshotRenamedTable.
*
* @param volume - volume name
* @param bucket - bucket name
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 9d24b6b880..fcd3b2cde2 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -187,7 +187,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
* |----------------------------------------------------------------------|
* | openFileTable | /volumeId/bucketId/parentId/fileName/id -> KeyInfo|
* |----------------------------------------------------------------------|
- * | deletedDirTable | /volumeId/bucketId/parentId/dirName -> KeyInfo |
+ * | deletedDirTable | /volumeId/bucketId/parentId/dirName/objectId -> |
+ * | | KeyInfo |
* |----------------------------------------------------------------------|
*
* Snapshot Tables:
@@ -196,7 +197,10 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
*
|-------------------------------------------------------------------------|
* | snapshotInfoTable | /volume/bucket/snapshotName -> SnapshotInfo
|
*
|-------------------------------------------------------------------------|
- * |snapshotRenamedKeyTable| /volumeName/bucketName/objectID -> /v/b/origKey
|
+ * | snapshotRenamedTable | /volumeName/bucketName/objectID -> One of:
|
+ * | | 1. /volumeId/bucketId/parentId/dirName
|
+ * | | 2. /volumeId/bucketId/parentId/fileName
|
+ * | | 3. /volumeName/bucketName/keyName
|
*
|-------------------------------------------------------------------------|
*/
@@ -224,8 +228,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
"principalToAccessIdsTable";
public static final String TENANT_STATE_TABLE = "tenantStateTable";
public static final String SNAPSHOT_INFO_TABLE = "snapshotInfoTable";
- public static final String SNAPSHOT_RENAMED_KEY_TABLE =
- "snapshotRenamedKeyTable";
+ public static final String SNAPSHOT_RENAMED_TABLE =
+ "snapshotRenamedTable";
static final String[] ALL_TABLES = new String[] {
USER_TABLE,
@@ -248,7 +252,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
PRINCIPAL_TO_ACCESS_IDS_TABLE,
TENANT_STATE_TABLE,
SNAPSHOT_INFO_TABLE,
- SNAPSHOT_RENAMED_KEY_TABLE
+ SNAPSHOT_RENAMED_TABLE
};
private DBStore store;
@@ -277,7 +281,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
private Table tenantStateTable;
private Table snapshotInfoTable;
- private Table snapshotRenamedKeyTable;
+ private Table snapshotRenamedTable;
private boolean isRatisEnabled;
private boolean ignorePipelineinKey;
@@ -576,7 +580,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
.addTable(PRINCIPAL_TO_ACCESS_IDS_TABLE)
.addTable(TENANT_STATE_TABLE)
.addTable(SNAPSHOT_INFO_TABLE)
- .addTable(SNAPSHOT_RENAMED_KEY_TABLE)
+ .addTable(SNAPSHOT_RENAMED_TABLE)
.addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec())
.addCodec(OmKeyInfo.class, new OmKeyInfoCodec(true))
.addCodec(RepeatedOmKeyInfo.class,
@@ -697,12 +701,12 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
String.class, SnapshotInfo.class);
checkTableStatus(snapshotInfoTable, SNAPSHOT_INFO_TABLE, addCacheMetrics);
- // volumeName/bucketName/objectID -> renamedKey (renamed key for key table)
- snapshotRenamedKeyTable = this.store.getTable(SNAPSHOT_RENAMED_KEY_TABLE,
+ // volumeName/bucketName/objectID -> renamedKey or renamedDir
+ snapshotRenamedTable = this.store.getTable(SNAPSHOT_RENAMED_TABLE,
String.class, String.class);
- checkTableStatus(snapshotRenamedKeyTable, SNAPSHOT_RENAMED_KEY_TABLE,
+ checkTableStatus(snapshotRenamedTable, SNAPSHOT_RENAMED_TABLE,
addCacheMetrics);
- // TODO: [SNAPSHOT] Initialize table lock for snapshotRenamedKeyTable.
+ // TODO: [SNAPSHOT] Initialize table lock for snapshotRenamedTable.
}
/**
@@ -1786,8 +1790,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
}
@Override
- public Table<String, String> getSnapshotRenamedKeyTable() {
- return snapshotRenamedKeyTable;
+ public Table<String, String> getSnapshotRenamedTable() {
+ return snapshotRenamedTable;
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index 0b5885e4bf..7a78821e9d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@ -231,22 +231,22 @@ public class OMDBDefinition implements DBDefinition {
new OmDBSnapshotInfoCodec());
/**
- * SnapshotRenamedKeyTable that complements the keyTable (or fileTable)
- * entries of the immediately previous snapshot in the same snapshot
- * scope (bucket or volume).
+ * SnapshotRenamedTable that complements the keyTable (or fileTable)
+ * and dirTable entries of the immediately previous snapshot in the
+ * same snapshot scope (bucket or volume).
* <p>
- * Key renames between the two subsequent snapshots are captured, this
+ * Key/Dir renames between the two subsequent snapshots are captured, this
* information is used in {@link SnapshotDeletingService} to check if the
- * renamedKey is present in the previous snapshot's keyTable
+ * renamedKey or renamedDir is present in the previous snapshot's keyTable
* (or fileTable).
*/
public static final DBColumnFamilyDefinition<String, String>
- SNAPSHOT_RENAMED_KEY_TABLE =
+ SNAPSHOT_RENAMED_TABLE =
new DBColumnFamilyDefinition<>(
- OmMetadataManagerImpl.SNAPSHOT_RENAMED_KEY_TABLE,
+ OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE,
String.class, // /volumeName/bucketName/objectID
new StringCodec(),
- String.class, // path to the key in previous snapshot's
key(file)Table
+ String.class, // path to key in prev snapshot's key(file)/dir Table.
new StringCodec());
@Override
@@ -268,7 +268,7 @@ public class OMDBDefinition implements DBDefinition {
FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, META_TABLE,
TENANT_ACCESS_ID_TABLE,
PRINCIPAL_TO_ACCESS_IDS_TABLE, TENANT_STATE_TABLE,
- SNAPSHOT_INFO_TABLE, SNAPSHOT_RENAMED_KEY_TABLE};
+ SNAPSHOT_INFO_TABLE, SNAPSHOT_RENAMED_TABLE};
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java
index a890ab5c67..9a97a3ea05 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java
@@ -31,12 +31,12 @@ import java.io.IOException;
import javax.annotation.Nonnull;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE;
-import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_RENAMED_KEY_TABLE;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE;
/**
* Response for RenameKey request.
*/
-@CleanupTableInfo(cleanupTables = {KEY_TABLE, SNAPSHOT_RENAMED_KEY_TABLE})
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, SNAPSHOT_RENAMED_TABLE})
public class OMKeyRenameResponse extends OmKeyResponse {
private String fromKeyName;
@@ -85,16 +85,16 @@ public class OMKeyRenameResponse extends OmKeyResponse {
renameKeyInfo);
// Check if the bucket is in snapshot scope, if yes
- // add the key to snapshotRenamedKeyTable.
+ // add the key to snapshotRenamedTable.
boolean isSnapshotBucket = OMClientRequestUtils.
isSnapshotBucket(omMetadataManager, renameKeyInfo);
String renameDbKey = omMetadataManager.getRenameKey(
renameKeyInfo.getVolumeName(), renameKeyInfo.getBucketName(),
renameKeyInfo.getObjectID());
- String renamedKey = omMetadataManager.getSnapshotRenamedKeyTable()
+ String renamedKey = omMetadataManager.getSnapshotRenamedTable()
.get(renameDbKey);
if (isSnapshotBucket && renamedKey == null) {
- omMetadataManager.getSnapshotRenamedKeyTable().putWithBatch(
+ omMetadataManager.getSnapshotRenamedTable().putWithBatch(
batchOperation, renameDbKey, fromDbKey);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java
index 3e3efbd4d1..5daff92e74 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java
@@ -34,13 +34,13 @@ import java.io.IOException;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE;
-import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_RENAMED_KEY_TABLE;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE;
/**
* Response for RenameKey request - prefix layout.
*/
@CleanupTableInfo(cleanupTables = {FILE_TABLE, DIRECTORY_TABLE,
- SNAPSHOT_RENAMED_KEY_TABLE})
+ SNAPSHOT_RENAMED_TABLE})
public class OMKeyRenameResponseWithFSO extends OMKeyRenameResponse {
private boolean isRenameDirectory;
@@ -91,20 +91,19 @@ public class OMKeyRenameResponseWithFSO extends
OMKeyRenameResponse {
.deleteWithBatch(batchOperation, getFromKeyName());
omMetadataManager.getKeyTable(getBucketLayout())
.putWithBatch(batchOperation, getToKeyName(), getRenameKeyInfo());
+ }
- boolean isSnapshotBucket = OMClientRequestUtils.
- isSnapshotBucket(omMetadataManager, getRenameKeyInfo());
- String renameDbKey = omMetadataManager.getRenameKey(
- getRenameKeyInfo().getVolumeName(),
- getRenameKeyInfo().getBucketName(),
- getRenameKeyInfo().getObjectID());
-
- String renamedKey = omMetadataManager.getSnapshotRenamedKeyTable()
- .get(renameDbKey);
- if (isSnapshotBucket && renamedKey == null) {
- omMetadataManager.getSnapshotRenamedKeyTable().putWithBatch(
- batchOperation, renameDbKey, getFromKeyName());
- }
+ boolean isSnapshotBucket = OMClientRequestUtils.
+ isSnapshotBucket(omMetadataManager, getRenameKeyInfo());
+ String renameDbKey = omMetadataManager.getRenameKey(
+ getRenameKeyInfo().getVolumeName(),
+ getRenameKeyInfo().getBucketName(),
+ getRenameKeyInfo().getObjectID());
+ String renamedKey = omMetadataManager.getSnapshotRenamedTable()
+ .get(renameDbKey);
+ if (isSnapshotBucket && renamedKey == null) {
+ omMetadataManager.getSnapshotRenamedTable().putWithBatch(
+ batchOperation, renameDbKey, getFromKeyName());
}
if (fromKeyParent != null) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
index 7625410bb6..f0ea2922a7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
@@ -32,14 +32,14 @@ import java.io.IOException;
import java.util.Map;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE;
-import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_RENAMED_KEY_TABLE;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_RENAME;
/**
* Response for RenameKeys request.
*/
-@CleanupTableInfo(cleanupTables = {KEY_TABLE, SNAPSHOT_RENAMED_KEY_TABLE})
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, SNAPSHOT_RENAMED_TABLE})
public class OMKeysRenameResponse extends OMClientResponse {
private OmRenameKeys omRenameKeys;
@@ -96,10 +96,10 @@ public class OMKeysRenameResponse extends OMClientResponse {
newKeyInfo.getVolumeName(), newKeyInfo.getBucketName(),
newKeyInfo.getObjectID());
- String renamedKey = omMetadataManager.getSnapshotRenamedKeyTable()
+ String renamedKey = omMetadataManager.getSnapshotRenamedTable()
.get(renameDbKey);
if (isSnapshotBucket && renamedKey == null) {
- omMetadataManager.getSnapshotRenamedKeyTable().putWithBatch(
+ omMetadataManager.getSnapshotRenamedTable().putWithBatch(
batchOperation, renameDbKey, fromDbKey);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
index 9745e438e9..fd4c11b862 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
@@ -33,14 +33,14 @@ import java.io.IOException;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE;
-import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_RENAMED_KEY_TABLE;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE;
import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE;
/**
* Response for OMSnapshotCreateRequest.
*/
@CleanupTableInfo(cleanupTables = {
- DELETED_TABLE, SNAPSHOT_RENAMED_KEY_TABLE, SNAPSHOT_INFO_TABLE})
+ DELETED_TABLE, SNAPSHOT_RENAMED_TABLE, SNAPSHOT_INFO_TABLE})
public class OMSnapshotCreateResponse extends OMClientResponse {
private SnapshotInfo snapshotInfo;
@@ -82,9 +82,9 @@ public class OMSnapshotCreateResponse extends
OMClientResponse {
snapshotInfo);
// TODO: [SNAPSHOT] Move to createOmSnapshotCheckpoint and add table lock
- // Remove all entries from snapshotRenamedKeyTable
+ // Remove all entries from snapshotRenamedTable
try (TableIterator<String, ? extends Table.KeyValue<String, String>>
- iterator = omMetadataManager.getSnapshotRenamedKeyTable().iterator()) {
+ iterator = omMetadataManager.getSnapshotRenamedTable().iterator()) {
String dbSnapshotBucketKey = omMetadataManager.getBucketKey(
snapshotInfo.getVolumeName(), snapshotInfo.getBucketName())
@@ -96,7 +96,7 @@ public class OMSnapshotCreateResponse extends
OMClientResponse {
if (!renameDbKey.startsWith(dbSnapshotBucketKey)) {
break;
}
- omMetadataManager.getSnapshotRenamedKeyTable()
+ omMetadataManager.getSnapshotRenamedTable()
.deleteWithBatch(batchOperation, renameDbKey);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
index f09e6dbd13..6c89828e3b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
@@ -126,7 +126,7 @@ public class OMSnapshotMoveDeletedKeysResponse extends
OMClientResponse {
// Move renamed keys to only the next snapshot or active DB.
if (isNextDB) {
for (HddsProtos.KeyValue renamedKey: renamedKeysList) {
- metadataManager.getSnapshotRenamedKeyTable()
+ metadataManager.getSnapshotRenamedTable()
.putWithBatch(batchOp, renamedKey.getKey(), renamedKey.getValue());
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
index 40caf08954..f859b67ec7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
@@ -156,8 +156,8 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
Table<String, OmKeyInfo> snapshotDeletedDirTable =
omSnapshot.getMetadataManager().getDeletedDirTable();
- Table<String, String> renamedKeyTable =
- omSnapshot.getMetadataManager().getSnapshotRenamedKeyTable();
+ Table<String, String> renamedTable =
+ omSnapshot.getMetadataManager().getSnapshotRenamedTable();
long volumeId = ozoneManager.getMetadataManager()
.getVolumeId(snapInfo.getVolumeName());
@@ -209,12 +209,13 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
// or keep it in current snapshot deleted table.
List<SnapshotMoveKeyInfos> toReclaimList = new ArrayList<>();
List<SnapshotMoveKeyInfos> toNextDBList = new ArrayList<>();
- List<HddsProtos.KeyValue> renamedKeysList = new ArrayList<>();
+ // A list of renamed keys/files/dirs
+ List<HddsProtos.KeyValue> renamedList = new ArrayList<>();
List<String> dirsToMove = new ArrayList<>();
long remainNum = handleDirectoryCleanUp(snapshotDeletedDirTable,
- previousDirTable, dbBucketKeyForDir, snapInfo, omSnapshot,
- dirsToMove);
+ previousDirTable, renamedTable, dbBucketKeyForDir, snapInfo,
+ omSnapshot, dirsToMove, renamedList);
int deletionCount = 0;
try (TableIterator<String, ? extends Table.KeyValue<String,
@@ -250,7 +251,7 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) {
splitRepeatedOmKeyInfo(toReclaim, toNextDb, renamedKey,
- keyInfo, previousKeyTable, renamedKeyTable,
+ keyInfo, previousKeyTable, renamedTable,
bucketInfo, volumeId);
}
@@ -271,7 +272,7 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
}
if (renamedKey.hasKey() && renamedKey.hasValue()) {
- renamedKeysList.add(renamedKey.build());
+ renamedList.add(renamedKey.build());
}
deletionCount++;
}
@@ -290,7 +291,7 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
snapshotLimit--;
// Submit Move request to OM.
submitSnapshotMoveDeletedKeys(snapInfo, toReclaimList,
- toNextDBList, renamedKeysList, dirsToMove);
+ toNextDBList, renamedList, dirsToMove);
}
submitSnapshotPurgeRequest(purgeSnapshotKeys);
} catch (IOException e) {
@@ -325,12 +326,14 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
(isKeyTableCleanedUp || snapshotDeletedTable.isEmpty());
}
+ @SuppressWarnings("checkstyle:ParameterNumber")
private long handleDirectoryCleanUp(
Table<String, OmKeyInfo> snapshotDeletedDirTable,
Table<String, OmDirectoryInfo> previousDirTable,
- String dbBucketKeyForDir,
- SnapshotInfo snapInfo, OmSnapshot omSnapshot,
- List<String> dirsToMove) {
+ Table<String, String> renamedTable,
+ String dbBucketKeyForDir, SnapshotInfo snapInfo,
+ OmSnapshot omSnapshot, List<String> dirsToMove,
+ List<HddsProtos.KeyValue> renamedList) {
long dirNum = 0L;
long subDirNum = 0L;
@@ -350,7 +353,8 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
Table.KeyValue<String, OmKeyInfo> deletedDir =
deletedDirIterator.next();
- if (checkDirReclaimable(deletedDir, previousDirTable)) {
+ if (checkDirReclaimable(deletedDir, previousDirTable,
+ renamedTable, renamedList)) {
// Reclaim here
PurgePathRequest request = prepareDeleteDirRequest(
remainNum, deletedDir.getValue(), deletedDir.getKey(),
@@ -406,10 +410,10 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
SnapshotMoveKeyInfos.Builder toNextDb,
HddsProtos.KeyValue.Builder renamedKey, OmKeyInfo keyInfo,
Table<String, OmKeyInfo> previousKeyTable,
- Table<String, String> renamedKeyTable,
+ Table<String, String> renamedTable,
OmBucketInfo bucketInfo, long volumeId) throws IOException {
- if (checkKeyReclaimable(previousKeyTable, renamedKeyTable,
+ if (checkKeyReclaimable(previousKeyTable, renamedTable,
keyInfo, bucketInfo, volumeId, renamedKey)) {
// Move to next non deleted snapshot's deleted table
toNextDb.addKeyInfos(keyInfo.getProtobuf(
@@ -424,7 +428,7 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo,
List<SnapshotMoveKeyInfos> toReclaimList,
List<SnapshotMoveKeyInfos> toNextDBList,
- List<HddsProtos.KeyValue> renamedKeysList,
+ List<HddsProtos.KeyValue> renamedList,
List<String> dirsToMove) {
SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder =
@@ -434,7 +438,7 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
SnapshotMoveDeletedKeysRequest moveDeletedKeys = moveDeletedKeysBuilder
.addAllReclaimKeys(toReclaimList)
.addAllNextDBKeys(toNextDBList)
- .addAllRenamedKeys(renamedKeysList)
+ .addAllRenamedKeys(renamedList)
.addAllDeletedDirsToMove(dirsToMove)
.build();
@@ -449,7 +453,9 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
private boolean checkDirReclaimable(
Table.KeyValue<String, OmKeyInfo> deletedDir,
- Table<String, OmDirectoryInfo> previousDirTable) throws IOException {
+ Table<String, OmDirectoryInfo> previousDirTable,
+ Table<String, String> renamedTable,
+ List<HddsProtos.KeyValue> renamedList) throws IOException {
if (previousDirTable == null) {
return true;
@@ -457,10 +463,32 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
String deletedDirDbKey = deletedDir.getKey();
OmKeyInfo deletedDirInfo = deletedDir.getValue();
- // In OMKeyDeleteResponseWithFSO OzonePathKey is converted to
- // OzoneDeletePathKey. Changing it back to check the previous DirTable.
- String prevDbKey = ozoneManager.getMetadataManager()
- .getOzoneDeletePathDirKey(deletedDirDbKey);
+ String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey(
+ deletedDirInfo.getVolumeName(), deletedDirInfo.getBucketName(),
+ deletedDirInfo.getObjectID());
+
+ /*
+ snapshotRenamedTable: /volumeName/bucketName/objectID ->
+ /volumeId/bucketId/parentId/dirName
+ */
+ String dbKeyBeforeRename = renamedTable.getIfExist(dbRenameKey);
+ String prevDbKey = null;
+
+ if (dbKeyBeforeRename != null) {
+ prevDbKey = dbKeyBeforeRename;
+ HddsProtos.KeyValue renamedDir = HddsProtos.KeyValue
+ .newBuilder()
+ .setKey(dbRenameKey)
+ .setValue(dbKeyBeforeRename)
+ .build();
+ renamedList.add(renamedDir);
+ } else {
+ // In OMKeyDeleteResponseWithFSO OzonePathKey is converted to
+ // OzoneDeletePathKey. Changing it back to check the previous DirTable.
+ prevDbKey = ozoneManager.getMetadataManager()
+ .getOzoneDeletePathDirKey(deletedDirDbKey);
+ }
+
OmDirectoryInfo prevDirectoryInfo = previousDirTable.get(prevDbKey);
if (prevDirectoryInfo == null) {
return true;
@@ -471,7 +499,7 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
private boolean checkKeyReclaimable(
Table<String, OmKeyInfo> previousKeyTable,
- Table<String, String> renamedKeyTable,
+ Table<String, String> renamedTable,
OmKeyInfo deletedKeyInfo, OmBucketInfo bucketInfo,
long volumeId, HddsProtos.KeyValue.Builder renamedKeyBuilder)
throws IOException {
@@ -502,7 +530,7 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
}
/*
- snapshotRenamedKeyTable:
+ snapshotRenamedTable:
1) /volumeName/bucketName/objectID ->
/volumeId/bucketId/parentId/fileName (FSO)
2) /volumeName/bucketName/objectID ->
@@ -512,10 +540,10 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
deletedKeyInfo.getVolumeName(), deletedKeyInfo.getBucketName(),
deletedKeyInfo.getObjectID());
- // Condition: key should not exist in snapshotRenamedKeyTable
+ // Condition: key should not exist in snapshotRenamedTable
// of the current snapshot and keyTable of the previous snapshot.
- // Check key exists in renamedKeyTable of the Snapshot
- String renamedKey = renamedKeyTable.getIfExist(dbRenameKey);
+ // Check key exists in renamedTable of the Snapshot
+ String renamedKey = renamedTable.getIfExist(dbRenameKey);
if (renamedKey != null) {
renamedKeyBuilder.setKey(dbRenameKey).setValue(renamedKey);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
index 2ef6965fd6..a67814156d 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.AuditMessage;
@@ -34,10 +35,12 @@ import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponseWithFSO;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
import org.apache.ozone.test.LambdaTestUtils;
import org.junit.After;
@@ -252,11 +255,14 @@ public class TestOMSnapshotCreateRequest {
@Test
public void testEmptySnapshotRenamedKeyTable() throws Exception {
when(ozoneManager.isAdmin(any())).thenReturn(true);
+ Table<String, String> snapshotRenamedTable =
+ omMetadataManager.getSnapshotRenamedTable();
- renameKey("key1", "key2");
+ renameKey("key1", "key2", 0);
+ renameDir("dir1", "dir2", 5);
// Rename table should be empty as there is no rename happening in
// the snapshot scope.
-
Assert.assertTrue(omMetadataManager.getSnapshotRenamedKeyTable().isEmpty());
+ Assert.assertTrue(snapshotRenamedTable.isEmpty());
// Create snapshot
createSnapshot(snapshotName1);
@@ -266,15 +272,16 @@ public class TestOMSnapshotCreateRequest {
omMetadataManager.getSnapshotInfoTable().get(snapKey);
Assert.assertNotNull(snapshotInfo);
- renameKey("key3", "key4");
- // Rename table should have one entry as rename is within snapshot scope.
- Assert.assertFalse(omMetadataManager.getSnapshotRenamedKeyTable()
- .isEmpty());
+ renameKey("key3", "key4", 10);
+ renameDir("dir3", "dir4", 15);
- // Create snapshot to clear snapshotRenamedKeyTable
- createSnapshot(snapshotName2);
-
Assert.assertTrue(omMetadataManager.getSnapshotRenamedKeyTable().isEmpty());
+ // Rename table should have two entries as rename is within snapshot scope.
+ Assert.assertEquals(2, omMetadataManager
+ .countRowsInTable(snapshotRenamedTable));
+ // Create snapshot to clear snapshotRenamedTable
+ createSnapshot(snapshotName2);
+ Assert.assertTrue(snapshotRenamedTable.isEmpty());
}
@Test
@@ -311,9 +318,10 @@ public class TestOMSnapshotCreateRequest {
omResponse.getStatus());
}
- private void renameKey(String fromKey, String toKey) throws IOException {
- OmKeyInfo toKeyInfo = addKey(fromKey);
- OmKeyInfo fromKeyInfo = addKey(toKey);
+ private void renameKey(String fromKey, String toKey, long offset)
+ throws IOException {
+ OmKeyInfo toKeyInfo = addKey(toKey, offset + 1L);
+ OmKeyInfo fromKeyInfo = addKey(fromKey, offset + 2L);
OMResponse omResponse = OMResponse
.newBuilder()
@@ -326,11 +334,42 @@ public class TestOMSnapshotCreateRequest {
new OMKeyRenameResponse(omResponse, fromKeyInfo.getKeyName(),
toKeyInfo.getKeyName(), toKeyInfo);
-
Assert.assertTrue(omMetadataManager.getSnapshotRenamedKeyTable().isEmpty());
omKeyRenameResponse.addToDBBatch(omMetadataManager, batchOperation);
omMetadataManager.getStore().commitBatchOperation(batchOperation);
}
+ private void renameDir(String fromKey, String toKey, long offset)
+ throws Exception {
+ String fromKeyParentName = UUID.randomUUID().toString();
+ OmKeyInfo fromKeyParent = OMRequestTestUtils.createOmKeyInfo(volumeName,
+ bucketName, fromKeyParentName, HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE, 100L);
+
+ OmKeyInfo toKeyInfo = addKey(toKey, offset + 4L);
+ OmKeyInfo fromKeyInfo = addKey(fromKey, offset + 5L);
+ OMResponse omResponse = OMResponse
+ .newBuilder()
+ .setRenameKeyResponse(
+ OzoneManagerProtocolProtos.RenameKeyResponse.getDefaultInstance())
+ .setStatus(OzoneManagerProtocolProtos.Status.OK)
+ .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
+ .build();
+
+ OMKeyRenameResponseWithFSO omKeyRenameResponse =
+ new OMKeyRenameResponseWithFSO(omResponse, getDBKeyName(fromKeyInfo),
+ getDBKeyName(toKeyInfo), fromKeyParent, null, toKeyInfo,
+ null, true, BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ omKeyRenameResponse.addToDBBatch(omMetadataManager, batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+ }
+
+ protected String getDBKeyName(OmKeyInfo keyInfo) throws IOException {
+ return omMetadataManager.getOzonePathKey(
+ omMetadataManager.getVolumeId(volumeName),
+ omMetadataManager.getBucketId(volumeName, bucketName),
+ keyInfo.getParentObjectID(), keyInfo.getKeyName());
+ }
+
private void createSnapshot(String snapName) throws Exception {
OMRequest omRequest =
OMRequestTestUtils.createSnapshotRequest(
@@ -362,9 +401,10 @@ public class TestOMSnapshotCreateRequest {
return new OMSnapshotCreateRequest(modifiedRequest);
}
- private OmKeyInfo addKey(String keyName) {
+ private OmKeyInfo addKey(String keyName, long objectId) {
return OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
- HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE,
0L);
+ HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE,
+ objectId);
}
protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java
index 5fab714b22..fd8839d8c9 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java
@@ -63,7 +63,7 @@ public class TestOMKeyRenameResponse extends
TestOMKeyResponse {
.isExist(dbFromKey));
Assert.assertFalse(omMetadataManager.getKeyTable(getBucketLayout())
.isExist(dbToKey));
-
Assert.assertTrue(omMetadataManager.getSnapshotRenamedKeyTable().isEmpty());
+ Assert.assertTrue(omMetadataManager.getSnapshotRenamedTable().isEmpty());
if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
Assert.assertFalse(omMetadataManager.getDirectoryTable()
.isExist(getDBKeyName(fromKeyParent)));
@@ -86,9 +86,9 @@ public class TestOMKeyRenameResponse extends
TestOMKeyResponse {
String renameDbKey = omMetadataManager.getRenameKey(
fromKeyInfo.getVolumeName(), fromKeyInfo.getBucketName(),
fromKeyInfo.getObjectID());
- // snapshotRenamedKeyTable shouldn't contain those keys which
+ // snapshotRenamedTable shouldn't contain those keys which
// is not part of snapshot bucket.
- Assert.assertFalse(omMetadataManager.getSnapshotRenamedKeyTable()
+ Assert.assertFalse(omMetadataManager.getSnapshotRenamedTable()
.isExist(renameDbKey));
if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
